Upload File from Local to S3 Bucket using CURL

 

Upload data to S3 bucket using CURL.

This guide aka shell script will help you to upload files into S3 without installing AWS SDK, Python Boto or AWS CLI.

GitHub URL

Script's README section has most of the usage defined, high level things script can do.

• Copy files from a specific directory between specific range dates.
• Copy specific files with search filer.
• Script logs all the file copy into file with time stamp. 
• Reinitiate is supported without re-write

#!/bin/bash
#
# Parameters
# $1 => Directory/Folder to search file.
# $2 => AWS Bucket subdirectories 
#       Example -- myAWSs3bucket/folderA/FolderB/FolderC
#              1.) In case one want to put files in folderA, use folderA as $2
#                  2.) In case one want to put files in folderB, use folderA/folderB as $2
#                  3.) In case one want to put files in folderC, use folderA/folderB/folderC as $2
# $3 => Existense of file from Start date in format YYYYMMDD 
#       Example --
#                  1.) 20210104 -> 4th January 2021
#                  2.) 20201212 -> 12th December 2020
# $4 => Existense of file upto end date in format YYYYMMDD
#       Example --
#                  1.) 20200322 -> 22nd March 2020
#                  2.) 20201212 -> 12th December 2020
# $5 => File Filter 
#       Example -- We need only specific files from a folder.
#                  1.) 20200122_data_settlement.txt --> Use $5 as *_data_settlement.txt
#                  2.) salesdata-20201215100610.txt --> Use $5 as salesdata-*
#      
# 
# Task - Find similar 20200122_data_settlement.txt on location /usr/data/
#        File existence date range 20200322 (22nd March 2020) to 20210104 (4th January 2021)
#        Copy it to AWS S3 bucket's subfolder named as folderA 
#
#     
# Syntax -  ./copy_data_to_S3_via_Curl.sh <LocalFolderLocation> <S3BUCKET-DIRECTORY> <STARTDATE> <ENDDATE> <FILEFILTER>
#
# Usage
#
#        1.) With File Filter
#         ./copy_data_to_S3_via_Curl.sh /usr/data folderA 20200322 20210104  '*data_settlement.txt'
#
#        2.) Without File Filter
#         ./copy_data_to_S3_via_Curl.sh /usr/data folderA 20200322 20210104  
#        
#    3.) Reinitiate left upload
#
#         ./copy_data_to_S3_via_Curl.sh 1 folderA
#
#
#  Flow 
#  1.) Script use find command to find all the files with parameters and write it to a file "/tmp/file_inventory.txt"
#  2.) For Loop is being used further ti read file inputs and do S3 operations using HTTPS API
#  3.) Script keeps removing the entries from inventory file after a successful upload.
#  4.) Script writes the successful and failed upload status within log file "/tmp/file_copy_status.log"
#  5.) Incase we want to interrupt and upload the remaining files later, comment line no 62
#        62 find $1 -newermt $3 \! -newermt $4   -iname "$5" >> $inventory
#      To avoid confusion run the script with same paramter.
#
#
# Author: Jackuna
#

# Bucket Data
bucket="mys3bucket-data"
s3_access_key="AKgtusjksskXXXXTQTW"
s3_secret_key="KSKKSIS HSNKSLS+ydRQ3Ya37A5NUd1V7QvEwDUZR"

# Files
inventory="/tmp/file_inventory.txt"
logme="/tmp/file_copy_status.log"


if  [ $# == 2 ]; then
  echo "`date` -  Initiating left file upload from old inventory " >> $logme

elif [ $# -eq 5 ]; then
  truncate -s 0 $inventory
  find $1 -newermt $3 \! -newermt $4   -iname "$5" >> $inventory
  echo "`date` - Initiating all file that contains string $5 and found between $3 - $4  upload from new inventory " >> $logme

elif [ $# -eq 4 ]; then
  truncate -s 0 $inventory
  find $1 -newermt $3 \! -newermt $4  >> $inventory
  sed -i 1d $inventory
  echo "`date` - Initiating all file found between $3 - $4  upload from new inventory " >> $logme

else
  echo " Some or all arguments Missing from CLI"
  echo " Usage :  ./copy_data_to_S3_via_Curl.sh <LocalFolderLocation> <S3BUCKET-DIRECTORY> <STARTDATE> <ENDDATE> <FILEFILTER>"
  echo " Open Script README section"
  exit 1
fi

file_list=`cat $inventory`
total_file_count=`cat $inventory|wc -l`


for local_file_val in $file_list; do
        aws_folder=$2
        aws_file_name=`echo $local_file_val| rev| cut -d '/' -f1 | rev`
        aws_filepath="/${bucket}/$aws_folder/$aws_file_name"

        # metadata
        contentType="application/x-compressed-tar"
        dateValue=`date -R`
        signature_string="PUT\n\n${contentType}\n${dateValue}\n${aws_filepath}"
        signature_hash=`echo -en ${signature_string} | openssl sha1 -hmac ${s3_secret_key} -binary | base64`


        curl -X PUT -T "$local_file_val" \
    -H "Host: ${bucket}.s3.amazonaws.com" \
    -H "Date: ${dateValue}" \
    -H "Content-Type: ${contentType}" \
    -H "Authorization: AWS ${s3_access_key}:${signature_hash}" \
        https://${bucket}.s3.amazonaws.com/$aws_folder/$aws_file_name

    if [ $? -gt 0 ]; then
            echo "`date` Upload Failed  $local_file_val to $bucket" >> $logme
    else
            echo "`date` Upload Success $local_file_val to $bucket" >> $logme
            count=$((count + 1))
            printf "\rCopy Status -  $count/$total_file_count - Completed "

            sleep 1
            sed -i "/\/$aws_file_name/d" $inventory
    fi

done;

Feel free to comment.