Upload data to S3 bucket using CURL.
This guide aka shell script will help you to upload files into S3 without installing AWS SDK, Python Boto or AWS CLI.
Script's README section has most of the usage defined, high level things script can do.
- Copy files from a specific directory between specific range dates.
- Copy specific files with search filer.
- Script logs all the file copy into file with time stamp.
- Reinitiate is supported without re-write
#!/bin/bash## Parameters# $1 => Directory/Folder to search file.# $2 => AWS Bucket subdirectories # Example -- myAWSs3bucket/folderA/FolderB/FolderC# 1.) In case one want to put files in folderA, use folderA as $2# 2.) In case one want to put files in folderB, use folderA/folderB as $2# 3.) In case one want to put files in folderC, use folderA/folderB/folderC as $2# $3 => Existense of file from Start date in format YYYYMMDD # Example --# 1.) 20210104 -> 4th January 2021# 2.) 20201212 -> 12th December 2020# $4 => Existense of file upto end date in format YYYYMMDD# Example --# 1.) 20200322 -> 22nd March 2020# 2.) 20201212 -> 12th December 2020# $5 => File Filter # Example -- We need only specific files from a folder.# 1.) 20200122_data_settlement.txt --> Use $5 as *_data_settlement.txt# 2.) salesdata-20201215100610.txt --> Use $5 as salesdata-*# # # Task - Find similar 20200122_data_settlement.txt on location /usr/data/# File existence date range 20200322 (22nd March 2020) to 20210104 (4th January 2021)# Copy it to AWS S3 bucket's subfolder named as folderA ## # Syntax - ./copy_data_to_S3_via_Curl.sh <LocalFolderLocation> <S3BUCKET-DIRECTORY> <STARTDATE> <ENDDATE> <FILEFILTER>## Usage## 1.) With File Filter# ./copy_data_to_S3_via_Curl.sh /usr/data folderA 20200322 20210104 '*data_settlement.txt'## 2.) Without File Filter# ./copy_data_to_S3_via_Curl.sh /usr/data folderA 20200322 20210104 # # 3.) Reinitiate left upload## ./copy_data_to_S3_via_Curl.sh 1 folderA### Flow # 1.) Script use find command to find all the files with parameters and write it to a file "/tmp/file_inventory.txt"# 2.) For Loop is being used further ti read file inputs and do S3 operations using HTTPS API# 3.) Script keeps removing the entries from inventory file after a successful upload.# 4.) Script writes the successful and failed upload status within log file "/tmp/file_copy_status.log"# 5.) Incase we want to interrupt and upload the remaining files later, comment line no 62# 62 find $1 -newermt $3 \! -newermt $4 -iname "$5" >> $inventory# To avoid confusion run the script with same paramter.### Author: Jackuna#
# Bucket Databucket="mys3bucket-data"s3_access_key="AKgtusjksskXXXXTQTW"s3_secret_key="KSKKSIS HSNKSLS+ydRQ3Ya37A5NUd1V7QvEwDUZR"
# Filesinventory="/tmp/file_inventory.txt"logme="/tmp/file_copy_status.log"
if [ $# == 2 ]; then echo "`date` - Initiating left file upload from old inventory " >> $logme
elif [ $# -eq 5 ]; then truncate -s 0 $inventory find $1 -newermt $3 \! -newermt $4 -iname "$5" >> $inventory echo "`date` - Initiating all file that contains string $5 and found between $3 - $4 upload from new inventory " >> $logme
elif [ $# -eq 4 ]; then truncate -s 0 $inventory find $1 -newermt $3 \! -newermt $4 >> $inventory sed -i 1d $inventory echo "`date` - Initiating all file found between $3 - $4 upload from new inventory " >> $logme
else echo " Some or all arguments Missing from CLI" echo " Usage : ./copy_data_to_S3_via_Curl.sh <LocalFolderLocation> <S3BUCKET-DIRECTORY> <STARTDATE> <ENDDATE> <FILEFILTER>" echo " Open Script README section" exit 1fi
file_list=`cat $inventory`total_file_count=`cat $inventory|wc -l`
for local_file_val in $file_list; do aws_folder=$2 aws_file_name=`echo $local_file_val| rev| cut -d '/' -f1 | rev` aws_filepath="/${bucket}/$aws_folder/$aws_file_name"
# metadata contentType="application/x-compressed-tar" dateValue=`date -R` signature_string="PUT\n\n${contentType}\n${dateValue}\n${aws_filepath}" signature_hash=`echo -en ${signature_string} | openssl sha1 -hmac ${s3_secret_key} -binary | base64`
curl -X PUT -T "$local_file_val" \ -H "Host: ${bucket}.s3.amazonaws.com" \ -H "Date: ${dateValue}" \ -H "Content-Type: ${contentType}" \ -H "Authorization: AWS ${s3_access_key}:${signature_hash}" \ https://${bucket}.s3.amazonaws.com/$aws_folder/$aws_file_name
if [ $? -gt 0 ]; then echo "`date` Upload Failed $local_file_val to $bucket" >> $logme else echo "`date` Upload Success $local_file_val to $bucket" >> $logme count=$((count + 1)) printf "\rCopy Status - $count/$total_file_count - Completed "
sleep 1 sed -i "/\/$aws_file_name/d" $inventory fi
done;
#!/bin/bash## Parameters# $1 => Directory/Folder to search file.# $2 => AWS Bucket subdirectories# Example -- myAWSs3bucket/folderA/FolderB/FolderC# 1.) In case one want to put files in folderA, use folderA as $2# 2.) In case one want to put files in folderB, use folderA/folderB as $2# 3.) In case one want to put files in folderC, use folderA/folderB/folderC as $2# $3 => Existense of file from Start date in format YYYYMMDD# Example --# 1.) 20210104 -> 4th January 2021# 2.) 20201212 -> 12th December 2020# $4 => Existense of file upto end date in format YYYYMMDD# Example --# 1.) 20200322 -> 22nd March 2020# 2.) 20201212 -> 12th December 2020# $5 => File Filter# Example -- We need only specific files from a folder.# 1.) 20200122_data_settlement.txt --> Use $5 as *_data_settlement.txt# 2.) salesdata-20201215100610.txt --> Use $5 as salesdata-*### Task - Find similar 20200122_data_settlement.txt on location /usr/data/# File existence date range 20200322 (22nd March 2020) to 20210104 (4th January 2021)# Copy it to AWS S3 bucket's subfolder named as folderA### Syntax - ./copy_data_to_S3_via_Curl.sh <LocalFolderLocation> <S3BUCKET-DIRECTORY> <STARTDATE> <ENDDATE> <FILEFILTER>## Usage## 1.) With File Filter# ./copy_data_to_S3_via_Curl.sh /usr/data folderA 20200322 20210104 '*data_settlement.txt'## 2.) Without File Filter# ./copy_data_to_S3_via_Curl.sh /usr/data folderA 20200322 20210104## 3.) Reinitiate left upload## ./copy_data_to_S3_via_Curl.sh 1 folderA### Flow# 1.) Script use find command to find all the files with parameters and write it to a file "/tmp/file_inventory.txt"# 2.) For Loop is being used further ti read file inputs and do S3 operations using HTTPS API# 3.) Script keeps removing the entries from inventory file after a successful upload.# 4.) Script writes the successful and failed upload status within log file "/tmp/file_copy_status.log"# 5.) Incase we want to interrupt and upload the remaining files later, comment line no 62# 62 find $1 -newermt $3 \! -newermt $4 -iname "$5" >> $inventory# To avoid confusion run the script with same paramter.### Author: Jackuna## Bucket Databucket="mys3bucket-data"s3_access_key="AKgtusjksskXXXXTQTW"s3_secret_key="KSKKSIS HSNKSLS+ydRQ3Ya37A5NUd1V7QvEwDUZR"# Filesinventory="/tmp/file_inventory.txt"logme="/tmp/file_copy_status.log"if [ $# == 2 ]; thenecho "`date` - Initiating left file upload from old inventory " >> $logmeelif [ $# -eq 5 ]; thentruncate -s 0 $inventoryfind $1 -newermt $3 \! -newermt $4 -iname "$5" >> $inventoryecho "`date` - Initiating all file that contains string $5 and found between $3 - $4 upload from new inventory " >> $logmeelif [ $# -eq 4 ]; thentruncate -s 0 $inventoryfind $1 -newermt $3 \! -newermt $4 >> $inventorysed -i 1d $inventoryecho "`date` - Initiating all file found between $3 - $4 upload from new inventory " >> $logmeelseecho " Some or all arguments Missing from CLI"echo " Usage : ./copy_data_to_S3_via_Curl.sh <LocalFolderLocation> <S3BUCKET-DIRECTORY> <STARTDATE> <ENDDATE> <FILEFILTER>"echo " Open Script README section"exit 1fifile_list=`cat $inventory`total_file_count=`cat $inventory|wc -l`for local_file_val in $file_list; doaws_folder=$2aws_file_name=`echo $local_file_val| rev| cut -d '/' -f1 | rev`aws_filepath="/${bucket}/$aws_folder/$aws_file_name"# metadatacontentType="application/x-compressed-tar"dateValue=`date -R`signature_string="PUT\n\n${contentType}\n${dateValue}\n${aws_filepath}"signature_hash=`echo -en ${signature_string} | openssl sha1 -hmac ${s3_secret_key} -binary | base64`curl -X PUT -T "$local_file_val" \-H "Host: ${bucket}.s3.amazonaws.com" \-H "Date: ${dateValue}" \-H "Content-Type: ${contentType}" \-H "Authorization: AWS ${s3_access_key}:${signature_hash}" \https://${bucket}.s3.amazonaws.com/$aws_folder/$aws_file_nameif [ $? -gt 0 ]; thenecho "`date` Upload Failed $local_file_val to $bucket" >> $logmeelseecho "`date` Upload Success $local_file_val to $bucket" >> $logmecount=$((count + 1))printf "\rCopy Status - $count/$total_file_count - Completed "sleep 1sed -i "/\/$aws_file_name/d" $inventoryfidone;
