Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Centralized upload #520

Merged
merged 18 commits into from
Oct 1, 2021
Merged

Centralized upload #520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
d88371c
Implement centralized upload
EricClaeys Sep 29, 2021
6fa7b39
Centralized Upload
EricClaeys Sep 29, 2021
66e3e57
Centralized upload
EricClaeys Sep 29, 2021
ccd5d0c
Centralized upload
EricClaeys Sep 29, 2021
57be9e5
Centralized upload
EricClaeys Sep 29, 2021
e6c3b2e
Centralized upload
EricClaeys Sep 30, 2021
343d6b5
Centralized upload
EricClaeys Sep 30, 2021
bb6ad28
Centralize upload
EricClaeys Sep 30, 2021
192b1a5
Centralize upload
EricClaeys Sep 30, 2021
4e6ab34
Centralize upload
EricClaeys Sep 30, 2021
e43c19d
Fixed IMAGES_DIR in copy_notification_image.sh
EricClaeys Sep 30, 2021
412fd78
change spaces to tabs, endOfNight.sh
EricClaeys Sep 30, 2021
c041e01
convert spaces to tabs, timelapse.sh
EricClaeys Sep 30, 2021
cbbd057
postData.sh: set correct timezone
EricClaeys Oct 1, 2021
46a5f85
generateForDay: made requested changes
EricClaeys Oct 1, 2021
74d2cfe
saveImageDay: made requested updates
EricClaeys Oct 1, 2021
2b80cb6
upload.sh: s/jgp/jpg/
EricClaeys Oct 1, 2021
e5bb846
generateForDay.sh
EricClaeys Oct 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 45 additions & 53 deletions scripts/copy_notification_image.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,85 +1,77 @@
#!/bin/bash

ME="$(basename "$BASH_ARGV0")" # Include script name in output so it's easier to find in the log file
ME="$(basename "${BASH_ARGV0}")"

NOTIFICATIONFILE="$1" # filename, minus the extension, since the extension may vary
if [ "$1" = "" ] ; then
echo "*** $ME: ERROR: no file specified" >&2
echo "*** ${ME}: ERROR: no file specified" >&2
exit 1
fi

source $ALLSKY_HOME/config.sh
source $ALLSKY_HOME/scripts/filename.sh
source $ALLSKY_HOME/scripts/ftp-settings.sh
source "${ALLSKY_HOME}/variables.sh"
source "${ALLSKY_CONFIG}/config.sh"
source "${ALLSKY_SCRIPTS}/filename.sh"
source "${ALLSKY_SCRIPTS}/ftp-settings.sh"

cd $ALLSKY_HOME
cd "${ALLSKY_HOME}"

NOTIFICATIONFILE="notification_images/$NOTIFICATIONFILE.$EXTENSION"
if [ ! -e "$NOTIFICATIONFILE" ] ; then
echo "*** $ME: ERROR: File '$NOTIFICATIONFILE' does not exist or is empty!" >&2
exit 1
NOTIFICATIONFILE="${ALLSKY_NOTIFICATION_IMAGES}/${NOTIFICATIONFILE}.${EXTENSION}"
if [ ! -e "${NOTIFICATIONFILE}" ] ; then
echo "*** ${ME}: ERROR: File '${NOTIFICATIONFILE}' does not exist or is empty!" >&2
exit 2
fi

IMAGE_TO_USE="$FULL_FILENAME"
cp "$NOTIFICATIONFILE" "$IMAGE_TO_USE" # don't overwrite notification image
IMAGE_TO_USE="${ALLSKY_TMP}/notification-${FULL_FILENAME}"
# Don't overwrite notification image so create a temporary copy and use that.
cp "${NOTIFICATIONFILE}" "${IMAGE_TO_USE}"

# Resize the image if required
if [ $IMG_RESIZE = "true" ]; then
convert "$IMAGE_TO_USE" -resize "$IMG_WIDTH"x"$IMG_HEIGHT" "$IMAGE_TO_USE"
if [ "${IMG_RESIZE}" = "true" ]; then
convert "${IMAGE_TO_USE}" -resize "${IMG_WIDTH}x${IMG_HEIGHT}" "${IMAGE_TO_USE}"
RET=$?
if [ $RET -ne 0 ] ; then
echo "*** $ME: ERROR: IMG_RESIZE failed with RET=$RET"
exit 1
if [ ${RET} -ne 0 ] ; then
echo "*** ${ME}: ERROR: IMG_RESIZE failed with RET=${RET}"
exit 3
fi
fi

# IMG_DIR and IMG_PREFIX are in config.sh
# If the user specified an IMG_PREFIX, copy the file to that name so the websites can display it.
if [ "${IMG_PREFIX}" != "" ]; then
cp "$IMAGE_TO_USE" "${IMG_PREFIX}${FILENAME}.${EXTENSION}"
fi

# If 24 hour saving is desired, save the image in today's thumbnail directory
# If daytime saving is desired, save the image in today's thumbnail directory
# so the user can see when things changed.
# Don't save in main image directory because we don't want the notification image in timelapses.
if [ "$CAPTURE_24HR" = "true" ] ; then
CURRENT=$(date +'%Y%m%d')
mkdir -p images/$CURRENT
THUMBNAILS_DIR=images/$CURRENT/thumbnails
mkdir -p $THUMBNAILS_DIR
# If at nighttime, save them in (possibly) yesterday's directory.
# If during day, save in today's directory.
if [ "${CAPTURE_24HR}" = "true" ] ; then
IMAGES_DIR="${ALLSKY_IMAGES}/$(date +'%Y%m%d')"
THUMB="${IMAGES_DIR}/thumbnails/${FILENAME}-$(date +'%Y%m%d%H%M%S').${EXTENSION}"

SAVED_FILE="$FILENAME-$(date +'%Y%m%d%H%M%S').$EXTENSION"
# Create a thumbnail of the image for faster load in web GUI
convert "$IMAGE_TO_USE" -resize "${THUMBNAIL_SIZE_X}x${THUMBNAIL_SIZE_Y}" "$THUMBNAILS_DIR/$SAVED_FILE"
convert "${IMAGE_TO_USE}" -resize "${THUMBNAIL_SIZE_X}x${THUMBNAIL_SIZE_Y}" "${THUMB}"
RET=$?
if [ $RET -ne 0 ] ; then
echo "*** $ME: WARNING: THUMBNAIL resize failed with RET=$RET; continuing."
if [ ${RET} -ne 0 ] ; then
echo "*** ${ME}: WARNING: THUMBNAIL resize failed with RET=${RET}; continuing."
fi
fi

mv -f "${IMAGE_TO_USE}" "${ALLSKY_HOME}/${FULL_FILENAME}" # so web severs can see it.

# If upload is true, optionally create a smaller version of the image and upload it
if [ "$UPLOAD_IMG" = "true" ] ; then
if [ "$RESIZE_UPLOADS" = "true" ]; then
echo -e "$ME: Resizing $NOTIFICATIONFILE for uploading"
if [ "${UPLOAD_IMG}" = "true" ] ; then
# Don't overwrite FULL_FILENAME since the web server(s) may be looking at it.
cp "${ALLSKY_HOME}/${FULL_FILENAME}" "${ALLSKY_TMP}"
IMAGE_TO_USE="${ALLSKY_TMP}/${FULL_FILENAME}"
if [ "${RESIZE_UPLOADS}" = "true" ]; then
# Create a smaller version for upload
convert "$IMAGE_TO_USE" -resize "$RESIZE_UPLOADS_SIZE" -gravity East -chop 2x0 "$IMAGE_TO_USE"
convert "${IMAGE_TO_USE}" -resize "${RESIZE_UPLOADS_SIZE}" -gravity East -chop 2x0 "${IMAGE_TO_USE}"
RET=$?
if [ $RET -ne 0 ] ; then
echo "*** $ME: ERROR: RESIZE_UPLOADS failed with RET=$RET"
exit 1
if [ ${RET} -ne 0 ] ; then
echo "*** ${ME}: ERROR: RESIZE_UPLOADS failed with RET=${RET}"
exit 4
fi
fi

TS=$(ls -l --time-style='+%H:%M:%S' $IMAGE_TO_USE | awk '{print $6}')
echo -e "$ME: Uploading $(basename $NOTIFICATIONFILE) with timestamp: $TS\n"
if [ $PROTOCOL = "S3" ] ; then
$AWS_CLI_DIR/aws s3 cp "$IMAGE_TO_USE" s3://$S3_BUCKET$IMGDIR --acl $S3_ACL &
elif [ $PROTOCOL = "local" ] ; then
cp "$IMAGE_TO_USE" "$IMGDIR" &
else
TEMP_NAME="ni-$RANDOM"
# "ni" = notification image. Use unique temporary name.
lftp "$PROTOCOL://$USER:$PASSWORD@$HOST:$IMGDIR" -e "set net:max-retries 2; set net:timeout 20; put "$IMAGE_TO_USE" -o $TEMP_NAME; rm -f "$IMAGE_TO_USE"; mv $TEMP_NAME "$IMAGE_TO_USE"; bye" &

fi
# We're actually uploading $IMAGE_TO_USE, but show $NOTIFICATIONFILE
# in the message since it's more descriptive.
echo -e "${ME}: Uploading $(basename "${NOTIFICATIONFILE}")\n"
# NI == Notification Image
"${ALLSKY_SCRIPTS}/upload.sh" --silent "${IMAGE_TO_USE}" "${IMG_DIR}" "${FULL_FILENAME}" "NI"
fi
exit 0
202 changes: 113 additions & 89 deletions scripts/endOfNight.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,115 +1,139 @@
#!/bin/bash

ME="$(basename "${BASH_ARGV0}")"

if [ $# -eq 1 ] ; then
if [ "x$1" = "x-h" ] ; then
echo "Usage: $BASH_ARGV0 [YYYYmmdd]"
exit
else
LAST_NIGHT=$1
fi
if [ "${1}" = "-h" -o "${1}" = "--help" ] ; then
echo -e "${RED}Usage: ${ME} [YYYYmmdd]${NC}"
exit 0
else
DATE="${1}"
fi
else
LAST_NIGHT=$(date -d '12 hours ago' +'%Y%m%d')
DATE=$(date -d '12 hours ago' +'%Y%m%d')
fi

source $ALLSKY_HOME/config.sh
source $ALLSKY_HOME/scripts/filename.sh
source $ALLSKY_HOME/scripts/ftp-settings.sh
# Allow this script to be executed manually, which requires several variables to be set.
if [ -z "$ALLSKY_HOME" ] ; then
export ALLSKY_HOME=$(realpath $(dirname "${BASH_ARGV0}")/..)
fi

cd $ALLSKY_HOME/scripts
ME="$(basename "$BASH_ARGV0")" # Include script name in output so it's easier to find in the log file
source "${ALLSKY_HOME}/variables.sh"
source "${ALLSKY_CONFIG}/config.sh"
source "${ALLSKY_SCRIPTS}/filename.sh"
source "${ALLSKY_SCRIPTS}/ftp-settings.sh"

# Post end of night data. This includes next twilight time
if [[ $POST_END_OF_NIGHT_DATA == "true" ]]; then
echo -e "$ME: Posting next twilight time to let server know when to resume liveview\n"
./postData.sh
echo -e "\n"
DATE_DIR="${ALLSKY_IMAGES}/${DATE}"
if [ ! -d "${DATE_DIR}" ] ; then
echo -e "${ME}: ${RED}ERROR: '${DATE_DIR}' not found!${NC}"
exit 2
fi

LAST_NIGHT_DIR="$ALLSKY_HOME/images/$LAST_NIGHT"

# Scan for, and remove corrupt images before generating
# keograms and startrails. This can take several (tens of) minutes to run
# and isn't necessary unless your system produces corrupt images which then
# generate funny colors in the summary images...
if [[ "$REMOVE_BAD_IMAGES" == "true" ]]; then
echo -e "$ME: Removing bad images\n"
./removeBadImages.sh $LAST_NIGHT_DIR
# Post end of night data. This includes next twilight time
if [[ ${POST_END_OF_NIGHT_DATA} == "true" ]]; then
echo -e "${ME}: Posting next twilight time to let server know when to resume liveview\n"
"${ALLSKY_SCRIPTS}/postData.sh"
echo -e "\n"
fi

TMP_DIR="$ALLSKY_HOME/tmp"
mkdir -p "$TMP_DIR"
# Remove corrupt images before generating keograms and startrails.
# This can take several (tens of) minutes to run and isn't necessary unless the system
# produces corrupt images which then generate funny colors in the summary images.
if [[ "${REMOVE_BAD_IMAGES}" == "true" ]]; then
echo -e "${ME}: ===== Removing bad images"
"${ALLSKY_SCRIPTS}/removeBadImages.sh" "${DATE_DIR}"
echo
fi

# Generate keogram from collected images
if [[ $KEOGRAM == "true" ]]; then
echo -e "$ME: Generating Keogram\n"
mkdir -p $LAST_NIGHT_DIR/keogram/
OUTPUT="$LAST_NIGHT_DIR/keogram/keogram-$LAST_NIGHT.$EXTENSION"
${ALLSKY_HOME}/keogram -d $LAST_NIGHT_DIR/ -e $EXTENSION -o $OUTPUT
RETCODE=$?
if [[ $UPLOAD_KEOGRAM == "true" && $RETCODE = 0 ]] ; then
if [[ $PROTOCOL == "S3" ]] ; then
$AWS_CLI_DIR/aws s3 cp $OUTPUT s3://$S3_BUCKET$KEOGRAM_DIR --acl $S3_ACL &
elif [[ $PROTOCOL == "local" ]] ; then
cp $OUTPUT $KEOGRAM_DIR &
else
lftp "$PROTOCOL://$USER:$PASSWORD@$HOST:$KEOGRAM_DIR" \
-e "set net:max-retries 1; put "$OUTPUT"; bye" &
fi
fi

# Optionally copy to the local website in addition to the upload above.
if [ "$WEB_KEOGRAM_DIR" != "" ]; then
cp $OUTPUT "$WEB_KEOGRAM_DIR"
fi
echo -e "${ME}: ===== Generating Keogram"
mkdir -p "${DATE_DIR}/keogram/"
KEOGRAM_FILE="keogram-${DATE}.${EXTENSION}"
UPLOAD_FILE="${DATE_DIR}/keogram/${KEOGRAM_FILE}"

"${ALLSKY_HOME}/keogram" -d "${DATE_DIR}/" -e ${EXTENSION} -o "${UPLOAD_FILE}" ${KEOGRAM_PARAMETERS}
RETCODE=$?
if [[ ${UPLOAD_KEOGRAM} == "true" && ${RETCODE} = 0 ]] ; then
# If the user specified a different name for the destination file, use it.
if [ "${KEOGRAM_DESTINATION_NAME}" != "" ]; then
KEOGRAM_FILE="${KEOGRAM_DESTINATION_NAME}"
fi
# KG == KeoGram
"${ALLSKY_SCRIPTS}/upload.sh" "${UPLOAD_FILE}" "${KEOGRAM_DIR}" "${KEOGRAM_FILE}" "KG"

# Optionally copy to the local website in addition to the upload above.
if [ "${WEB_KEOGRAM_DIR}" != "" ]; then
echo "${ME}: Copying ${UPLOAD_FILE} to ${WEB_KEOGRAM_DIR}"
cp ${UPLOAD_FILE} "${WEB_KEOGRAM_DIR}"
fi
fi
echo
fi

# Generate startrails from collected images.
# Threshold set to 0.1 by default in config.sh to avoid stacking over-exposed images.
if [[ $STARTRAILS == "true" ]]; then
echo -e "$ME: Generating Startrails\n"
mkdir -p $LAST_NIGHT_DIR/startrails/
OUTPUT="$LAST_NIGHT_DIR/startrails/startrails-$LAST_NIGHT.$EXTENSION"
# The startrails command outputs one line for each of the many hundreds of files,
# and this adds needless clutter to the log file, so send output to a tmp file so we can output the
# number of images.
${ALLSKY_HOME}/startrails -d $LAST_NIGHT_DIR/ -e $EXTENSION -b $BRIGHTNESS_THRESHOLD -o $OUTPUT
RETCODE=$?
if [[ $UPLOAD_STARTRAILS == "true" && $RETCODE == 0 ]] ; then
if [[ $PROTOCOL == "S3" ]] ; then
$AWS_CLI_DIR/aws s3 cp $OUTPUT s3://$S3_BUCKET$STARTRAILS_DIR --acl $S3_ACL &
elif [[ $PROTOCOL == "local" ]] ; then
cp $OUTPUT $STARTRAILS_DIR &
else
lftp "$PROTOCOL"://"$USER":"$PASSWORD"@"$HOST":"$STARTRAILS_DIR" \
-e "set net:max-retries 1; put $OUTPUT; bye" &
fi
fi

# Optionally copy to the local website in addition to the upload above.
if [ "$WEB_STARTRAILS_DIR" != "" ]; then
cp $OUTPUT "$WEB_STARTRAILS_DIR"
fi
if [[ ${STARTRAILS} == "true" ]]; then
echo -e "${ME}: ===== Generating Startrails"
mkdir -p ${DATE_DIR}/startrails/
STARTRAILS_FILE="startrails-${DATE}.${EXTENSION}"
UPLOAD_FILE="${DATE_DIR}/startrails/${STARTRAILS_FILE}"
"${ALLSKY_HOME}/startrails" "${DATE_DIR}/" ${EXTENSION} ${BRIGHTNESS_THRESHOLD} "${UPLOAD_FILE}"
RETCODE=$?
if [[ ${UPLOAD_STARTRAILS} == "true" && ${RETCODE} == 0 ]] ; then
# If the user specified a different name for the destination file, use it.
if [ "${STARTRAILS_DESTINATION_NAME}" != "" ]; then
STARTRAILS_FILE="${STARTRAILS_DESTINATION_NAME}"
fi
# ST == Star Trails
"${ALLSKY_SCRIPTS}/upload.sh" "${UPLOAD_FILE}" "${STARTRAILS_DIR}" "${STARTRAILS_FILE}" "ST"

# Optionally copy to the local website in addition to the upload above.
if [ "${WEB_STARTRAILS_DIR}" != "" ]; then
echo "${ME}: Copying ${UPLOAD_FILE} to ${WEB_STARTRAILS_DIR}"
cp "${UPLOAD_FILE}" "${WEB_STARTRAILS_DIR}"
fi
fi
echo
fi

# Generate timelapse from collected images
if [[ $TIMELAPSE == "true" ]]; then
echo -e "$ME: Generating Timelapse\n"
./timelapse.sh $LAST_NIGHT
echo -e "\n"
# Generate timelapse from collected images.
# Use timelapse.sh instead of putting all the commands here so users can easily
# test the timelapse creation, which sometimes has issues.
if [[ ${TIMELAPSE} == "true" ]]; then
echo -e "${ME}: ===== Generating Timelapse"
"${ALLSKY_SCRIPTS}/timelapse.sh" "${DATE}"
RETCODE=$?
if [[ ${UPLOAD_VIDEO} == "true" && ${RETCODE} == 0 ]] ; then
VIDEOS_FILE="allsky-${DATE}.mp4"
UPLOAD_FILE="${DATE_DIR}/${VIDEOS_FILE}"
# If the user specified a different name for the destination file, use it.
if [ "${VIDEOS_DESTINATION_NAME}" != "" ]; then
VIDEOS_FILE="${VIDEOS_DESTINATION_NAME}"
fi
# TL == Time Lapse
"${ALLSKY_SCRIPTS}/upload.sh" "${UPLOAD_FILE}" "${VIDEOS_DIR}" "${VIDEOS_FILE}" "TL"

# Optionally copy to the local website in addition to the upload above.
if [ "$WEB_MP4DIR" != "" ]; then
cp $LAST_NIGHT_DIR/allsky-$LAST_NIGHT.mp4 "$WEB_MP4DIR"
fi
# Optionally copy to the local website in addition to the upload above.
if [ "${WEB_VIDEOS_DIR}" != "" ]; then
TIMELAPSE_FILE="allsky-${DATE}.mp4"
UPLOAD_FILE="${DATE_DIR}/${TIMELAPSE_FILE}"
echo "${ME}: Copying ${UPLOAD_FILE} to ${WEB_VIDEOS_DIR}"
cp "${UPLOAD_FILE}" "${WEB_VIDEOS_DIR}"
fi
fi
# timelapse.sh handled ${TMP}
fi

# Run custom script at the end of a night. This is run BEFORE the automatic deletion just in case you need to do something with the files before they are removed
./endOfNight_additionalSteps.sh
# Run custom script at the end of a night. This is run BEFORE the automatic deletion
# just in case you need to do something with the files before they are removed
"${ALLSKY_SCRIPTS}/endOfNight_additionalSteps.sh"

# Automatically delete old images and videos
if [[ $AUTO_DELETE == "true" ]]; then
del=$(date --date="$NIGHTS_TO_KEEP days ago" +%Y%m%d)
for i in `find $ALLSKY_HOME/images/ -type d -name "2*"`; do # "2*" for years >= 2000
(($del > $(basename $i))) && rm -rf $i
done
if [[ ${AUTO_DELETE} == "true" ]]; then
del=$(date --date="${NIGHTS_TO_KEEP} days ago" +%Y%m%d)
for i in $(find "${ALLSKY_IMAGES}/" -type d -name "2*"); do # "2*" for years >= 2000
((${del} > $(basename ${i}))) && echo "Deleting old directory ${i}" && rm -rf ${i}
done
fi

exit 0
Loading