Added Amazon S3 support for file uploading in ftp-settings-sh.repo, s…

…aveImageDay.sh, SaveImageNight.sh, endOfNight.sh, timelapse.sh, postData.sh, uploadForDay.sh, README.md.

Added asynchronous subshell to all lftp / s3 upload operations as was inconsistently used in some places but not others.

README.md

@@ -141,7 +141,7 @@ nano config.sh
 
 When using the cropping options the image is cropped from the center so you will need to experiment with the correct width and height values. Normally there will be no need to amend the offset values.
 
-In order to upload images and videos to your website, you'll need to fill your FTP connection details in **ftp-settings.sh**
+In order to upload images and videos to your website, you'll need to fill your FTP or Amazon S3 connection details in **ftp-settings.sh**
 ```shell
 nano scripts/ftp-settings.sh
 ```

scripts/endOfNight.sh

@@ -33,25 +33,33 @@ fi
 # Generate keogram from collected images
 if [[ $KEOGRAM == "true" ]]; then
         echo -e "Generating Keogram\n"
-	mkdir -p $ALLSKY_HOME/images/$LAST_NIGHT/keogram/
+        mkdir -p $ALLSKY_HOME/images/$LAST_NIGHT/keogram/
         ../keogram $ALLSKY_HOME/images/$LAST_NIGHT/ $EXTENSION $ALLSKY_HOME/images/$LAST_NIGHT/keogram/keogram-$LAST_NIGHT.$EXTENSION
-	if [[ $UPLOAD_KEOGRAM == "true" ]] ; then
-		OUTPUT="$ALLSKY_HOME/images/$LAST_NIGHT/keogram/keogram-$LAST_NIGHT.$EXTENSION"
-                lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$KEOGRAM_DIR" \
-                        -e "set net:max-retries 1; put $OUTPUT; bye"
-	fi
+        if [[ $UPLOAD_KEOGRAM == "true" ]] ; then
+                OUTPUT="$ALLSKY_HOME/images/$LAST_NIGHT/keogram/keogram-$LAST_NIGHT.$EXTENSION"
+                if [[ $PROTOCOL == "S3" ]] ; then
+                        $AWS_CLI_DIR/aws s3 cp $OUTPUT s3://$S3_BUCKET$KEOGRAM_DIR --acl $S3_ACL &
+                else
+                        lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$KEOGRAM_DIR" \
+                                -e "set net:max-retries 1; put $OUTPUT; bye" &
+                fi
+        fi
         echo -e "\n"
 fi
 
-# Generate startrails from collected images. Treshold set to 0.1 by default in config.sh to avoid stacking over-exposed images
+# Generate startrails from collected images. Threshold set to 0.1 by default in config.sh to avoid stacking over-exposed images
 if [[ $STARTRAILS == "true" ]]; then
         echo -e "Generating Startrails\n"
-	mkdir -p $ALLSKY_HOME/images/$LAST_NIGHT/startrails/
+        mkdir -p $ALLSKY_HOME/images/$LAST_NIGHT/startrails/
         ../startrails $ALLSKY_HOME/images/$LAST_NIGHT/ $EXTENSION $BRIGHTNESS_THRESHOLD $ALLSKY_HOME/images/$LAST_NIGHT/startrails/startrails-$LAST_NIGHT.$EXTENSION
-	if [[ $UPLOAD_STARTRAILS == "true" ]] ; then
-		OUTPUT="$ALLSKY_HOME/images/$LAST_NIGHT/startrails/startrails-$LAST_NIGHT.$EXTENSION"
-                lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$STARTRAILS_DIR" \
-			-e "set net:max-retries 1; put $OUTPUT; bye"
+        if [[ $UPLOAD_STARTRAILS == "true" ]] ; then
+                OUTPUT="$ALLSKY_HOME/images/$LAST_NIGHT/startrails/startrails-$LAST_NIGHT.$EXTENSION"
+                if [[ $PROTOCOL == "S3" ]] ; then
+                        $AWS_CLI_DIR/aws s3 cp $OUTPUT s3://$S3_BUCKET$STARTRAILS_DIR --acl $S3_ACL &
+                else
+                        lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$STARTRAILS_DIR" \
+                                -e "set net:max-retries 1; put $OUTPUT; bye" &
+                fi
         fi
 
         echo -e "\n"

scripts/ftp-settings.sh.repo

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# FTP/SFTP settings
+# FTP/SFTP/S3 settings
 PROTOCOL='ftp'
 USER='username'
 PASSWORD='password'
@@ -9,3 +9,22 @@ IMGDIR='/allsky/'
 MP4DIR='/allsky/videos/'
 KEOGRAM_DIR='/allsky/keograms/'
 STARTRAILS_DIR='/allsky/startrails/'
+
+# S3 settings (Set PROTOCOL='S3'above and configure DIR settings above also as required.)
+# You will need to install the AWS CLI:
+#   sudo apt-get install python3-pip
+#   pip3 install awscli --upgrade --user
+#   export PATH=/home/pi/.local/bin:$PATH
+#   aws configure
+# Enter a valid access key ID, Secret Access Key and Default region name (e.g. us-west-2) when prompted.
+# Set the Default output format to json when prompted
+#
+# AWS CLI directory where the AWS CLI tools are installed.
+AWS_CLI_DIR='/home/pi/.local/bin'
+# Name of S3 Bucket where the files are to be uploaded (must be in Default region specified above).
+# You may want to turn off or limit bucket versioning to avoid consuming lots of space with multiple versions of the live 'image-resize.*' file.
+S3_BUCKET='allskybucket'
+# S3_ACL is set to private by default. If you want to serve your uploaded files vis http(s), change S3_ACL to 'public-read'.  
+# You will need to ensure the S3 bucket policy is configured to allow public access to objects with a public-read ACL.
+# You may need to set a CORS policy in S3 if the files are to be accessed by Javascript from a different domain.
+S3_ACL='private'

scripts/postData.sh

@@ -23,4 +23,8 @@ echo \"sunset\": \"$today"T"$timeNoZone":00.000$timezone"\", >> data.json
 echo \"streamDaytime\": \"$streamDaytime\" >> data.json
 echo } >> data.json
 echo "Uploading data.json"
-lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$IMGDIR" -e "set net:max-retries 1; set net:timeout 20; put data.json; bye"
+if [[ $PROTOCOL == "S3" ]] ; then
+        $AWS_CLI_DIR/aws s3 cp data.json s3://$S3_BUCKET$IMGDIR --acl $S3_ACL &
+else
+        lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$IMGDIR" -e "set net:max-retries 1; set net:timeout 20; put data.json; bye" &
+fi

scripts/saveImageDay.sh

@@ -54,5 +54,9 @@ if [ "$UPLOAD_IMG" = true ] ; then
 
 	echo -e "Uploading\n"
 	echo -e "Uploading $FILENAME-resize.$EXTENSION \n" >> log.txt
-	lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$IMGDIR" -e "set net:max-retries 1; set net:timeout 20; put $FILENAME-resize.$EXTENSION; bye" &
+       if [[ $PROTOCOL == "S3" ]] ; then
+                $AWS_CLI_DIR/aws s3 cp $FILENAME-resize.$EXTENSION s3://$S3_BUCKET$IMGDIR --acl $S3_ACL &
+        else
+                lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$IMGDIR" -e "set net:max-retries 1; set net:timeout 20; put $FILENAME-resize.$EXTENSION; bye" $
+        fi
 fi

scripts/saveImageNight.sh

@@ -63,5 +63,9 @@ if [ "$UPLOAD_IMG" = true ] ; then
 
 	echo -e "Uploading \n"
 	echo -e "Uploading $FILENAME-resize.$EXTENSION \n" >> log.txt
-	lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$IMGDIR" -e "set net:max-retries 1; set net:timeout 20; put $FILENAME-resize.$EXTENSION; bye" &
+	if [[ $PROTOCOL == "S3" ]] ; then
+			$AWS_CLI_DIR/aws s3 cp $FILENAME-resize.$EXTENSION s3://$S3_BUCKET$IMGDIR --acl $S3_ACL &
+	else
+			lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$IMGDIR" -e "set net:max-retries 1; set net:timeout 20; put $FILENAME-resize.$EXTENSION; bye" &
+	fi
 fi

scripts/timelapse.sh

@@ -39,13 +39,17 @@ ffmpeg -y -f image2 \
 	-i images/$1/sequence/%04d.$EXTENSION \
 	-vcodec libx264 \
 	-b:v 2000k \
-	-pix_fmt yuv420p \
+	-pix_fmt yuv420p \./i
 	-movflags +faststart \
 	$SCALE \
 	images/$1/allsky-$1.mp4
 
 if [ "$UPLOAD_VIDEO" = true ] ; then
-	lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$MP4DIR" -e "set net:max-retries 1; put images/$1/allsky-$1.mp4; bye"
+        if [[ "$PROTOCOL" == "S3" ]] ; then
+                $AWS_CLI_DIR/aws s3 cp images/$1/allsky-$1.mp4 s3://$S3_BUCKET$MP4DIR --acl $S3_ACL &
+        else
+                lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$MP4DIR" -e "set net:max-retries 1; put images/$1/allsky-$1.mp4; bye" &
+        fi
 fi
 
 echo -en "* ${GREEN}Deleting sequence${NC}\n"

scripts/uploadForDay.sh

@@ -19,17 +19,29 @@ fi
 # Upload keogram
 echo -e "Uploading Keogram\n"
 KEOGRAM="$ALLSKY_HOME/images/$1/keogram/keogram-$1.$EXTENSION"
-lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$KEOGRAM_DIR" -e "set net:max-retries 1; put $KEOGRAM; bye" -u "$USER","$PASSWORD"
+if [[ $PROTOCOL == "S3" ]] ; then
+        $AWS_CLI_DIR/aws s3 cp $KEOGRAM s3://$S3_BUCKET$KEOGRAM_DIR --acl $S3_ACL &
+else
+        lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$KEOGRAM_DIR" -e "set net:max-retries 1; put $KEOGRAM; bye" -u "$USER","$PASSWORD" &
+fi
 echo -e "\n"
 
 # Upload Startrails
 echo -e "Uploading Startrails\n"
 STARTRAILS="$ALLSKY_HOME/images/$1/startrails/startrails-$1.$EXTENSION"
-lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$STARTRAILS_DIR" -e "set net:max-retries 1; put $STARTRAILS; bye"
+if [[ $PROTOCOL == "S3" ]] ; then
+        $AWS_CLI_DIR/aws s3 cp $STARTRAILS s3://$S3_BUCKET$STARTRAILS_DIR --acl $S3_ACL &
+else
+        lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$STARTRAILS_DIR" -e "set net:max-retries 1; put $STARTRAILS; bye" &
+fi
 echo -e "\n"
 
 # Upload timelapse
 echo -e "Uploading Timelapse\n"
 TIMELAPSE="$ALLSKY_HOME/images/$1/allsky-$1.mp4"
-lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$MP4DIR" -e "set net:max-retries 1; put $TIMELAPSE; bye"
-echo -e "\n"
+if [[ "$PROTOCOL" == "S3" ]] ; then
+        $AWS_CLI_DIR/aws s3 cp $TIMELAPSE s3://$S3_BUCKET$MP4DIR --acl $S3_ACL &
+else
+        lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$MP4DIR" -e "set net:max-retries 1; put $TIMELAPSE; bye" &
+fi
+echo -e "\n"