Add one_instance() function

scripts/functions.sh

@@ -100,7 +100,7 @@ function determineCommandToUse()
 		# the output of vcgencmd changes depending on the OS and how the Pi is configured.
 		# Newer kernels/libcamera give:   supported=1 detected=0, libcamera interfaces=1
 		# but only if    start_x=1    is in /boot/config.txt
-		vcgencmd get_camera | grep --silent "supported=1" ######### detected=1"
+		vcgencmd get_camera | /bin/grep --silent "supported=1" ######### detected=1"
 		RET=$?
 	fi
 
@@ -426,7 +426,7 @@ function get_variable() {
 	local FILE="${2}"
 	local LINE=""
 	local SEARCH_STRING="^[ 	]*${VARIABLE}="
-	if ! LINE="$(grep -E "${SEARCH_STRING}" "${FILE}")" ; then
+	if ! LINE="$( /bin/grep -E "${SEARCH_STRING}" "${FILE}" )" ; then
 		return 1
 	fi
 
@@ -562,3 +562,138 @@ function update_json_file()		# field, new value, file
 	# Have to use "cp" instead of "mv" to keep any hard link.
 	jq "${1} = \"${2}\"" "${FILE}" > "${TEMP}" && cp "${TEMP}" "${FILE}" && rm "${TEMP}"
 }
+
+####
+# Only allow one of the specified process at a time.
+function one_instance()
+{
+	local SLEEP_TIME="5s"
+	local MAX_CHECKS=3
+	local PROCESS_NAME=""
+	local PID_FILE=""
+	local ABORTED_FILE=""
+	local ABORTED_FIELDS=""
+	local ABORTED_MSG1=""
+	local ABORTED_MSG2=""
+
+	OK="true"
+	local ERRORS=""
+	while [[ $# -gt 0 ]]; do
+		ARG="${1}"
+		case "${ARG}" in
+				--sleep)
+					SLEEP_TIME="${2}"
+					shift
+					;;
+				--max-checks)
+					MAX_CHECKS=${2}
+					shift
+					;;
+				--process-name)
+					PROCESS_NAME="${2}"
+					shift
+					;;
+				--pid-file)
+					PID_FILE="${2}"
+					shift
+					;;
+				--aborted-count-file)
+					ABORTED_FILE="${2}"
+					shift
+					;;
+				--aborted-fields)
+					ABORTED_FIELDS="${2}"
+					shift
+					;;
+				--aborted-msg1)
+					ABORTED_MSG1="${2}"
+					shift
+					;;
+				--aborted-msg2)
+					ABORTED_MSG2="${2}"
+					shift
+					;;
+				*)
+					ERRORS="${ERRORS}\nUnknown argument: '${ARG}'."
+					OK="false"
+					;;
+		esac
+		shift
+	done
+	if [[ -z ${PROCESS_NAME} ]]; then
+		ERRORS="${ERRORS}\nPROCESS_NAME not specified."
+		OK="false"
+	fi
+	if [[ -z ${PID_FILE} ]]; then
+		ERRORS="${ERRORS}\nPID_FILE not specified."
+		OK="false"
+	fi
+	if [[ -z ${ABORTED_FILE} ]]; then
+		ERRORS="${ERRORS}\nABORTED_FILE not specified."
+		OK="false"
+	fi
+	if [[ -z ${ABORTED_FIELDS} ]]; then
+		ERRORS="${ERRORS}\nABORTED_FIELDS not specified."
+		OK="false"
+	fi
+	if [[ -z ${ABORTED_MSG1} ]]; then
+		ERRORS="${ERRORS}\nABORTED_MSG1 not specified."
+		OK="false"
+	fi
+	if [[ -z ${ABORTED_MSG2} ]]; then
+		ERRORS="${ERRORS}\nABORTED_MSG2 not specified."
+		OK="false"
+	fi
+
+	if [[ ${OK} == "false" ]]; then
+		echo -e "${RED}${ME}: ERROR: ${ERRORS}.${NC}" >&2
+		return 1
+	fi
+
+
+	NUM_CHECKS=0
+	while  : ; do
+		[[ ! -f ${PID_FILE} ]] && break
+
+		PID=$( < "${PID_FILE}" )
+		# shellcheck disable=SC2009
+		if ! ps -fp "${PID}" | /bin/grep --silent "${PROCESS_NAME}" ; then
+			break	# Not sure why the PID file existed if the process didn't exist.
+		fi
+
+		if [[ $NUM_CHECKS -eq ${MAX_CHECKS} ]]; then
+			echo -en "${YELLOW}" >&2
+			echo -e  "${ABORTED_MSG1}" >&2
+			echo -n  "Made ${NUM_CHECKS} attempts at waiting." >&2
+			echo -n  " If this happens often, check your settings." >&2
+			echo -e  "${NC}" >&2
+			ps -fp "${PID}" >&2
+
+			# Keep track of aborts so user can be notified.
+			# If it's happening often let the user know.
+			echo -e "$(date)\t${ABORTED_FIELDS}" >> "${ABORTED_FILE}"
+			NUM=$( wc -l < "${ABORTED_FILE}" )
+			if [[ ${NUM} -eq 3 || ${NUM} -eq 10 ]]; then
+				MSG="${NUM} ${ABORTED_MSG2} have been aborted waiting for others to finish."
+				MSG="${MSG}\nThis could be caused by a slow network or other network issues."
+				if [[ ${NUM} -eq 3 ]]; then
+					SEVERITY="info"
+				else
+					SEVERITY="warning"
+					MSG="${MSG}\nOnce you have resolved the cause, reset the aborted counter:"
+					MSG="${MSG}\n&nbsp; &nbsp; <code>rm -f '${ABORTED_FILE}'</code>"
+				fi
+				"${ALLSKY_SCRIPTS}/addMessage.sh" "${SEVERITY}" "${MSG}"
+			fi
+
+			return 2
+		else
+			sleep "${SLEEP_TIME}"
+		fi
+		((NUM_CHECKS++))
+	done
+
+	echo $$ > "${PID_FILE}" || return 1
+
+	return 0
+}

scripts/saveImage.sh

@@ -23,6 +23,7 @@ usage_and_exit()
 	exit ${retcode}
 }
 [[ $# -lt 2 ]] && usage_and_exit 1
+
 # Export so other scripts can use it.
 export DAY_OR_NIGHT="${1}"
 [[ ${DAY_OR_NIGHT} != "DAY" && ${DAY_OR_NIGHT} != "NIGHT" ]] && usage_and_exit 1
@@ -45,6 +46,20 @@ if [[ ! -s ${CURRENT_IMAGE} ]] ; then
 	exit 2
 fi
 
+# Make sure only one save happens at once.
+# Multiple concurrent saves (which can happen if the delay is short or post-processing
+# is long) causes read and write errors.
+PID_FILE="${ALLSKY_TMP}/saveImage-pid.txt"
+ABORTED_MSG1="Another saveImage is in progress so the new one was aborted."
+ABORTED_FIELDS="${CURRENT_IMAGE}"
+ABORTED_MSG2="uploads"
+if ! one_instance --process-name "${ME}" --pid-file "${PID_FILE}" \
+		--aborted-count-file "${ALLSKY_ABORTEDSAVEIMAGE:-/home/pi/allsky/tmp/aborted_saveImage.txt}" --aborted-fields "${ABORTED_FIELDS}" \
+		--aborted-msg1 "${ABORTED_MSG1}" --aborted-msg2 "${ABORTED_MSG2}" ; then
+	exit 1
+fi
+
+
 # The image may be in a memory filesystem, so do all the processing there and
 # leave the image used by the website(s) in that directory.
 IMAGE_NAME=$(basename "${CURRENT_IMAGE}")	# just the file name
@@ -225,6 +240,11 @@ fi
 
 "${ALLSKY_SCRIPTS}/flow-runner.py"
 
+# The majority of the post-processing time for an image is in flow-runner.py.
+# Since only one mini-timelapse can run at once and that code is embeded in this code
+# in several places, remove our PID lock now.
+rm -f "${PID_FILE}"
+
 SAVED_FILE="${CURRENT_IMAGE}"						# The name of the file saved from the camera.
 WEBSITE_FILE="${WORKING_DIR}/${FULL_FILENAME}"		# The name of the file the websites look for
 
@@ -310,17 +330,21 @@ if [[ ${SAVE_IMAGE} == "true" ]]; then
 				# shellcheck disable=SC2086
 				"${ALLSKY_SCRIPTS}"/timelapse.sh ${D} --mini "${MINI_TIMELAPSE_FILES}" "${DATE_NAME}"
 				RET=$?
-				[[ ${RET} -ne 0 ]] && TIMELAPSE_MINI_UPLOAD_VIDEO="false"			# failed so don't try to upload
+				if [[ ${RET} -ne 0 ]]; then
+					# failed so don't try to upload
+					TIMELAPSE_MINI_UPLOAD_VIDEO="false"
+				fi
 				if [[ ${ALLSKY_DEBUG_LEVEL} -ge 2 ]]; then
 					if [[ ${RET} -eq 0 ]]; then
-						echo "${ME}: mini-timelapse created"
+						echo "${ME}: mini-timelapse created (last image: ${IMAGE_NAME})"
 					else
-						echo "${ME}: mini-timelapse creation returned with RET=${RET}"
+						echo "${ME}: mini-timelapse creation returned with RET=${RET} (last image: ${IMAGE_NAME})"
 					fi
 				fi
 
-				# Remove the oldest files, but not if we only created this mini-timelapse because of a force
-				if [[ ${MOD} -ne 0 || ${TIMELAPSE_MINI_FORCE_CREATION} == "false" ]]; then
+				# Remove the oldest files, but not if we only created
+				# this mini-timelapse because of a force.
+				if [[ ${RET} -eq 0 && (${MOD} -ne 0 || ${TIMELAPSE_MINI_FORCE_CREATION} == "false") ]]; then
 					KEEP=$((TIMELAPSE_MINI_IMAGES - TIMELAPSE_MINI_FREQUENCY))
 					x="$(tail -${KEEP} "${MINI_TIMELAPSE_FILES}")"
 					echo -e "${x}" > "${MINI_TIMELAPSE_FILES}"

scripts/timelapse.sh

@@ -86,63 +86,22 @@ fi
 
 if [[ ${DO_MINI} == "false" ]]; then
 	OUTPUT_FILE="${DATE_DIR}/allsky-${DATE}.mp4"
+	SEQUENCE_DIR="${ALLSKY_TMP}/sequence-timelapse"
 else
 	# In MINI mode, only allow one process at a time.
 	OUTPUT_FILE="${ALLSKY_TMP}/mini-timelapse.mp4"
 	PID_FILE="${ALLSKY_TMP}/timelapse-mini-pid.txt"
-	NUM_CHECKS=0
-	MAX_CHECKS=2
-	SLEEP_TIME="15s"
-
-	while  : ; do
-		[[ ! -f ${PID_FILE} ]] && break
-
-		PID=$( < "${PID_FILE}" )
-		# shellcheck disable=SC2009
-		if ! ps -fp "${PID}" | grep -E --silent "${ME}.*--mini" ; then
-			break	# Not sure why the PID file existed if the process didn't exist.
-		fi
-
-		if [[ $NUM_CHECKS -eq ${MAX_CHECKS} ]]; then
-			echo -en "${YELLOW}" >&2
-			echo -n  "${ME}: WARNING: Another mini timelapse creation is in progress" >&2
-			echo     " so this one was aborted." >&2
-			echo     "If this happens often, check your network and delay settings" >&2
-			echo -n  "as well as your TIMELAPSE_MINI_IMAGES and TIMELAPSE_MINI_FREQUENCY settings." >&2
-			echo -e  "${NC}" >&2
-			ps -fp "${PID}" >&2
-
-			# Keep track of aborts so user can be notified.
-			# If it's happening often let the user know.
-			echo -e "$(date)\t${OUTPUT_FILE}" >> "${ALLSKY_ABORTEDTIMELAPSE}"
-			NUM=$( wc -l < "${ALLSKY_ABORTEDTIMELAPSE}" )
-			if [[ ${NUM} -eq 3 || ${NUM} -eq 10 ]]; then
-				MSG="${NUM} mini timelapse creations have been aborted waiting for others to finish."
-				MSG="${MSG}\nThis could be caused by unreasonable"
-				MSG="${MSG} TIMELAPSE_MINI_IMAGES and TIMELAPSE_MINI_FREQUENCY settings."
-				if [[ ${NUM} -eq 3 ]]; then
-					SEVERITY="info"
-				else
-					SEVERITY="warning"
-					MSG="${MSG}\nOnce you have resolved the cause, reset the aborted counter:"
-					MSG="${MSG}\n&nbsp; &nbsp; <code>rm -f '${ALLSKY_ABORTEDTIMELAPSE}'</code>"
-				fi
-				"${ALLSKY_SCRIPTS}/addMessage.sh" "${SEVERITY}" "${MSG}"
-			fi
-
-			exit 2
-		else
-			sleep "${SLEEP_TIME}"
-		fi
-		((NUM_CHECKS++))
-	done
-
-	echo $$ > "${PID_FILE}" || exit 1
+	ABORTED_MSG1="Another mini timelapse creation is in progress so this one was aborted."
+	ABORTED_FIELDS="${OUTPUT_FILE}"
+	ABORTED_MSG2="mini timelapse creations"
+	if ! one_instance --process-name "${ME}.*--mini" --pid-file "${PID_FILE}" \
+			--aborted-count-file "${ALLSKY_ABORTEDTIMELAPSE}" --aborted-fields "${ABORTED_FIELDS}" \
+			--aborted-msg1 "${ABORTED_MSG1}" --aborted-msg2 "${ABORTED_MSG2}" ; then
+		exit 1
+	fi
+	SEQUENCE_DIR="${ALLSKY_TMP}/sequence-mini-timelapse"
 fi
 
-# To save on writes to SD card for people who have $ALLSKY_TMP as a memory filesystem,
-# put the sequence files there.
-SEQUENCE_DIR="${ALLSKY_TMP}/sequence-${DATE}-$$"
 if [[ -d ${SEQUENCE_DIR} ]]; then
 	NSEQ=$(find "${SEQUENCE_DIR}/*" 2>/dev/null | wc -l)	# left over from last time
 else

scripts/upload.sh

@@ -115,56 +115,23 @@ LOG="${ALLSKY_TMP}/upload_errors.txt"
 # Multiple concurrent uploads (which can happen if the system and/or network is slow can
 # cause errors and files left on the server.
 PID_FILE="${ALLSKY_TMP}/${FILE_TYPE}-pid.txt"
-NUM_CHECKS=0
 if [[ ${WAIT} == "true" ]]; then
 	MAX_CHECKS=10
-	SLEEP_TIME="5s"
+	SLEEP="5s"
 else
 	MAX_CHECKS=2
-	SLEEP_TIME="10s"
+	SLEEP="10s"
+fi
+ABORTED_MSG1="Another '${FILE_TYPE}' upload is in progress so the new upload of"
+ABORTED_MSG1="${ABORTED_MSG1} $(basename "${FILE_TO_UPLOAD}") was aborted."
+ABORTED_FIELDS="${FILE_TYPE}\t${FILE_TO_UPLOAD}"
+ABORTED_MSG2="uploads"
+if ! one_instance --sleep "${SLEEP}" --max-checks "${MAX_CHECKS}" \
+		--process-name "${ME}" --pid-file "${PID_FILE}" \
+		--aborted-count-file "${ALLSKY_ABORTEDUPLOADS}" --aborted-fields "${ABORTED_FIELDS}" \
+		--aborted-msg1 "${ABORTED_MSG1}" --aborted-msg2 "${ABORTED_MSG2}" ; then
+	exit 1
 fi
-while  : ; do
-	[[ ! -f ${PID_FILE} ]] && break
-
-	PID=$( < "${PID_FILE}" )
-	# shellcheck disable=SC2009
-	if ! ps -p "${PID}" | grep --silent "${ME}" ; then
-		break	# Not sure why the PID file existed if the process didn't exist.
-	fi
-
-	if [[ $NUM_CHECKS -eq ${MAX_CHECKS} ]]; then
-		echo -en "${YELLOW}" >&2
-		echo -n  "${ME}: WARNING: Another '${FILE_TYPE}' upload is in" >&2
-		echo     " progress so the new upload of $(basename "${FILE_TO_UPLOAD}") was aborted." >&2
-		echo -n  "Made ${NUM_CHECKS} attempts at waiting." >&2
-		echo -n  " If this happens often, check your network and delay settings." >&2
-		echo -e  "${NC}" >&2
-		ps -fp "${PID}" >&2
-
-		# Keep track of aborts so user can be notified.
-		# If it's happening often let the user know.
-		echo -e "$(date)\t${FILE_TYPE}\t${FILE_TO_UPLOAD}" >> "${ALLSKY_ABORTEDUPLOADS}"
-		NUM=$( wc -l < "${ALLSKY_ABORTEDUPLOADS}" )
-		if [[ ${NUM} -eq 3 || ${NUM} -eq 10 ]]; then
-			MSG="${NUM} uploads have been aborted waiting for other uploads to finish."
-			MSG="${MSG}\nThis could be caused by a slow network or other network issues."
-			if [[ ${NUM} -eq 3 ]]; then
-				SEVERITY="info"
-			else
-				SEVERITY="warning"
-				MSG="${MSG}\nOnce you have resolved the cause, reset the aborted counter:"
-				MSG="${MSG}\n&nbsp; &nbsp; <code>rm -f '${ALLSKY_ABORTEDUPLOADS}'</code>"
-			fi
-			"${ALLSKY_SCRIPTS}/addMessage.sh" "${SEVERITY}" "${MSG}"
-		fi
-
-		exit 2
-	else
-		sleep "${SLEEP_TIME}"
-	fi
-	((NUM_CHECKS++))
-done
-echo $$ > "${PID_FILE}" || exit 1
 
 # Convert to lowercase so we don't care if user specified upper or lowercase.
 PROTOCOL="${PROTOCOL,,}"

variables.sh

@@ -82,9 +82,10 @@ if [[ -z "${ALLSKY_VARIABLE_SET}" ]]; then
 	ALLSKY_INSTALLATION_LOGS="${ALLSKY_CONFIG}/installation_logs"
 	POST_INSTALLATION_ACTIONS="${ALLSKY_INSTALLATION_LOGS}/post-installation_actions.txt"
 
-	# Holds temporary list of aborted uploads and timelapse since another one was in progress
+	# Holds temporary list of aborted processes since another one was in progress.
 	ALLSKY_ABORTEDUPLOADS="${ALLSKY_TMP}/aborted_uploads.txt"
 	ALLSKY_ABORTEDTIMELAPSE="${ALLSKY_TMP}/aborted_timelapse.txt"
+	ALLSKY_ABORTEDSAVEIMAGE="${ALLSKY_TMP}/aborted_saveImage.txt"
 
 	# Holds all the dark frames.
 	ALLSKY_DARKS="${ALLSKY_HOME}/darks"