diff --git a/scripts/functions.sh b/scripts/functions.sh index 4131075ff..6e0dac82f 100644 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -100,7 +100,7 @@ function determineCommandToUse() # the output of vcgencmd changes depending on the OS and how the Pi is configured. # Newer kernels/libcamera give: supported=1 detected=0, libcamera interfaces=1 # but only if start_x=1 is in /boot/config.txt - vcgencmd get_camera | grep --silent "supported=1" ######### detected=1" + vcgencmd get_camera | /bin/grep --silent "supported=1" ######### detected=1" RET=$? fi @@ -426,7 +426,7 @@ function get_variable() { local FILE="${2}" local LINE="" local SEARCH_STRING="^[ ]*${VARIABLE}=" - if ! LINE="$(grep -E "${SEARCH_STRING}" "${FILE}")" ; then + if ! LINE="$( /bin/grep -E "${SEARCH_STRING}" "${FILE}" )" ; then return 1 fi @@ -562,3 +562,138 @@ function update_json_file() # field, new value, file # Have to use "cp" instead of "mv" to keep any hard link. jq "${1} = \"${2}\"" "${FILE}" > "${TEMP}" && cp "${TEMP}" "${FILE}" && rm "${TEMP}" } + +#### +# Only allow one of the specified process at a time. +function one_instance() +{ + local SLEEP_TIME="5s" + local MAX_CHECKS=3 + local PROCESS_NAME="" + local PID_FILE="" + local ABORTED_FILE="" + local ABORTED_FIELDS="" + local ABORTED_MSG1="" + local ABORTED_MSG2="" + + OK="true" + local ERRORS="" + while [[ $# -gt 0 ]]; do + ARG="${1}" + case "${ARG}" in + --sleep) + SLEEP_TIME="${2}" + shift + ;; + --max-checks) + MAX_CHECKS=${2} + shift + ;; + --process-name) + PROCESS_NAME="${2}" + shift + ;; + --pid-file) + PID_FILE="${2}" + shift + ;; + --aborted-count-file) + ABORTED_FILE="${2}" + shift + ;; + --aborted-fields) + ABORTED_FIELDS="${2}" + shift + ;; + --aborted-msg1) + ABORTED_MSG1="${2}" + shift + ;; + --aborted-msg2) + ABORTED_MSG2="${2}" + shift + ;; + *) + ERRORS="${ERRORS}\nUnknown argument: '${ARG}'." + OK="false" + ;; + esac + shift + done + if [[ -z ${PROCESS_NAME} ]]; then + ERRORS="${ERRORS}\nPROCESS_NAME not specified." + OK="false" + fi + if [[ -z ${PID_FILE} ]]; then + ERRORS="${ERRORS}\nPID_FILE not specified." + OK="false" + fi + if [[ -z ${ABORTED_FILE} ]]; then + ERRORS="${ERRORS}\nABORTED_FILE not specified." + OK="false" + fi + if [[ -z ${ABORTED_FIELDS} ]]; then + ERRORS="${ERRORS}\nABORTED_FIELDS not specified." + OK="false" + fi + if [[ -z ${ABORTED_MSG1} ]]; then + ERRORS="${ERRORS}\nABORTED_MSG1 not specified." + OK="false" + fi + if [[ -z ${ABORTED_MSG2} ]]; then + ERRORS="${ERRORS}\nABORTED_MSG2 not specified." + OK="false" + fi + + if [[ ${OK} == "false" ]]; then + echo -e "${RED}${ME}: ERROR: ${ERRORS}.${NC}" >&2 + return 1 + fi + + + NUM_CHECKS=0 + while : ; do + [[ ! -f ${PID_FILE} ]] && break + + PID=$( < "${PID_FILE}" ) + # shellcheck disable=SC2009 + if ! ps -fp "${PID}" | /bin/grep --silent "${PROCESS_NAME}" ; then + break # Not sure why the PID file existed if the process didn't exist. + fi + + if [[ $NUM_CHECKS -eq ${MAX_CHECKS} ]]; then + echo -en "${YELLOW}" >&2 + echo -e "${ABORTED_MSG1}" >&2 + echo -n "Made ${NUM_CHECKS} attempts at waiting." >&2 + echo -n " If this happens often, check your settings." >&2 + echo -e "${NC}" >&2 + ps -fp "${PID}" >&2 + + # Keep track of aborts so user can be notified. + # If it's happening often let the user know. + echo -e "$(date)\t${ABORTED_FIELDS}" >> "${ABORTED_FILE}" + NUM=$( wc -l < "${ABORTED_FILE}" ) + if [[ ${NUM} -eq 3 || ${NUM} -eq 10 ]]; then + MSG="${NUM} ${ABORTED_MSG2} have been aborted waiting for others to finish." + MSG="${MSG}\nThis could be caused by a slow network or other network issues." + if [[ ${NUM} -eq 3 ]]; then + SEVERITY="info" + else + SEVERITY="warning" + MSG="${MSG}\nOnce you have resolved the cause, reset the aborted counter:" + MSG="${MSG}\n    rm -f '${ABORTED_FILE}'" + fi + "${ALLSKY_SCRIPTS}/addMessage.sh" "${SEVERITY}" "${MSG}" + fi + + return 2 + else + sleep "${SLEEP_TIME}" + fi + ((NUM_CHECKS++)) + done + + echo $$ > "${PID_FILE}" || return 1 + + return 0 +} diff --git a/scripts/saveImage.sh b/scripts/saveImage.sh index 51353c1f1..6336632ba 100755 --- a/scripts/saveImage.sh +++ b/scripts/saveImage.sh @@ -23,6 +23,7 @@ usage_and_exit() exit ${retcode} } [[ $# -lt 2 ]] && usage_and_exit 1 + # Export so other scripts can use it. export DAY_OR_NIGHT="${1}" [[ ${DAY_OR_NIGHT} != "DAY" && ${DAY_OR_NIGHT} != "NIGHT" ]] && usage_and_exit 1 @@ -45,6 +46,20 @@ if [[ ! -s ${CURRENT_IMAGE} ]] ; then exit 2 fi +# Make sure only one save happens at once. +# Multiple concurrent saves (which can happen if the delay is short or post-processing +# is long) causes read and write errors. +PID_FILE="${ALLSKY_TMP}/saveImage-pid.txt" +ABORTED_MSG1="Another saveImage is in progress so the new one was aborted." +ABORTED_FIELDS="${CURRENT_IMAGE}" +ABORTED_MSG2="uploads" +if ! one_instance --process-name "${ME}" --pid-file "${PID_FILE}" \ + --aborted-count-file "${ALLSKY_ABORTEDSAVEIMAGE:-/home/pi/allsky/tmp/aborted_saveImage.txt}" --aborted-fields "${ABORTED_FIELDS}" \ + --aborted-msg1 "${ABORTED_MSG1}" --aborted-msg2 "${ABORTED_MSG2}" ; then + exit 1 +fi + + # The image may be in a memory filesystem, so do all the processing there and # leave the image used by the website(s) in that directory. IMAGE_NAME=$(basename "${CURRENT_IMAGE}") # just the file name @@ -225,6 +240,11 @@ fi "${ALLSKY_SCRIPTS}/flow-runner.py" +# The majority of the post-processing time for an image is in flow-runner.py. +# Since only one mini-timelapse can run at once and that code is embeded in this code +# in several places, remove our PID lock now. +rm -f "${PID_FILE}" + SAVED_FILE="${CURRENT_IMAGE}" # The name of the file saved from the camera. WEBSITE_FILE="${WORKING_DIR}/${FULL_FILENAME}" # The name of the file the websites look for @@ -310,17 +330,21 @@ if [[ ${SAVE_IMAGE} == "true" ]]; then # shellcheck disable=SC2086 "${ALLSKY_SCRIPTS}"/timelapse.sh ${D} --mini "${MINI_TIMELAPSE_FILES}" "${DATE_NAME}" RET=$? - [[ ${RET} -ne 0 ]] && TIMELAPSE_MINI_UPLOAD_VIDEO="false" # failed so don't try to upload + if [[ ${RET} -ne 0 ]]; then + # failed so don't try to upload + TIMELAPSE_MINI_UPLOAD_VIDEO="false" + fi if [[ ${ALLSKY_DEBUG_LEVEL} -ge 2 ]]; then if [[ ${RET} -eq 0 ]]; then - echo "${ME}: mini-timelapse created" + echo "${ME}: mini-timelapse created (last image: ${IMAGE_NAME})" else - echo "${ME}: mini-timelapse creation returned with RET=${RET}" + echo "${ME}: mini-timelapse creation returned with RET=${RET} (last image: ${IMAGE_NAME})" fi fi - # Remove the oldest files, but not if we only created this mini-timelapse because of a force - if [[ ${MOD} -ne 0 || ${TIMELAPSE_MINI_FORCE_CREATION} == "false" ]]; then + # Remove the oldest files, but not if we only created + # this mini-timelapse because of a force. + if [[ ${RET} -eq 0 && (${MOD} -ne 0 || ${TIMELAPSE_MINI_FORCE_CREATION} == "false") ]]; then KEEP=$((TIMELAPSE_MINI_IMAGES - TIMELAPSE_MINI_FREQUENCY)) x="$(tail -${KEEP} "${MINI_TIMELAPSE_FILES}")" echo -e "${x}" > "${MINI_TIMELAPSE_FILES}" diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index 3308523d5..041962a6e 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -86,63 +86,22 @@ fi if [[ ${DO_MINI} == "false" ]]; then OUTPUT_FILE="${DATE_DIR}/allsky-${DATE}.mp4" + SEQUENCE_DIR="${ALLSKY_TMP}/sequence-timelapse" else # In MINI mode, only allow one process at a time. OUTPUT_FILE="${ALLSKY_TMP}/mini-timelapse.mp4" PID_FILE="${ALLSKY_TMP}/timelapse-mini-pid.txt" - NUM_CHECKS=0 - MAX_CHECKS=2 - SLEEP_TIME="15s" - - while : ; do - [[ ! -f ${PID_FILE} ]] && break - - PID=$( < "${PID_FILE}" ) - # shellcheck disable=SC2009 - if ! ps -fp "${PID}" | grep -E --silent "${ME}.*--mini" ; then - break # Not sure why the PID file existed if the process didn't exist. - fi - - if [[ $NUM_CHECKS -eq ${MAX_CHECKS} ]]; then - echo -en "${YELLOW}" >&2 - echo -n "${ME}: WARNING: Another mini timelapse creation is in progress" >&2 - echo " so this one was aborted." >&2 - echo "If this happens often, check your network and delay settings" >&2 - echo -n "as well as your TIMELAPSE_MINI_IMAGES and TIMELAPSE_MINI_FREQUENCY settings." >&2 - echo -e "${NC}" >&2 - ps -fp "${PID}" >&2 - - # Keep track of aborts so user can be notified. - # If it's happening often let the user know. - echo -e "$(date)\t${OUTPUT_FILE}" >> "${ALLSKY_ABORTEDTIMELAPSE}" - NUM=$( wc -l < "${ALLSKY_ABORTEDTIMELAPSE}" ) - if [[ ${NUM} -eq 3 || ${NUM} -eq 10 ]]; then - MSG="${NUM} mini timelapse creations have been aborted waiting for others to finish." - MSG="${MSG}\nThis could be caused by unreasonable" - MSG="${MSG} TIMELAPSE_MINI_IMAGES and TIMELAPSE_MINI_FREQUENCY settings." - if [[ ${NUM} -eq 3 ]]; then - SEVERITY="info" - else - SEVERITY="warning" - MSG="${MSG}\nOnce you have resolved the cause, reset the aborted counter:" - MSG="${MSG}\n    rm -f '${ALLSKY_ABORTEDTIMELAPSE}'" - fi - "${ALLSKY_SCRIPTS}/addMessage.sh" "${SEVERITY}" "${MSG}" - fi - - exit 2 - else - sleep "${SLEEP_TIME}" - fi - ((NUM_CHECKS++)) - done - - echo $$ > "${PID_FILE}" || exit 1 + ABORTED_MSG1="Another mini timelapse creation is in progress so this one was aborted." + ABORTED_FIELDS="${OUTPUT_FILE}" + ABORTED_MSG2="mini timelapse creations" + if ! one_instance --process-name "${ME}.*--mini" --pid-file "${PID_FILE}" \ + --aborted-count-file "${ALLSKY_ABORTEDTIMELAPSE}" --aborted-fields "${ABORTED_FIELDS}" \ + --aborted-msg1 "${ABORTED_MSG1}" --aborted-msg2 "${ABORTED_MSG2}" ; then + exit 1 + fi + SEQUENCE_DIR="${ALLSKY_TMP}/sequence-mini-timelapse" fi -# To save on writes to SD card for people who have $ALLSKY_TMP as a memory filesystem, -# put the sequence files there. -SEQUENCE_DIR="${ALLSKY_TMP}/sequence-${DATE}-$$" if [[ -d ${SEQUENCE_DIR} ]]; then NSEQ=$(find "${SEQUENCE_DIR}/*" 2>/dev/null | wc -l) # left over from last time else diff --git a/scripts/upload.sh b/scripts/upload.sh index 4fdf04b2c..e465b436e 100755 --- a/scripts/upload.sh +++ b/scripts/upload.sh @@ -115,56 +115,23 @@ LOG="${ALLSKY_TMP}/upload_errors.txt" # Multiple concurrent uploads (which can happen if the system and/or network is slow can # cause errors and files left on the server. PID_FILE="${ALLSKY_TMP}/${FILE_TYPE}-pid.txt" -NUM_CHECKS=0 if [[ ${WAIT} == "true" ]]; then MAX_CHECKS=10 - SLEEP_TIME="5s" + SLEEP="5s" else MAX_CHECKS=2 - SLEEP_TIME="10s" + SLEEP="10s" +fi +ABORTED_MSG1="Another '${FILE_TYPE}' upload is in progress so the new upload of" +ABORTED_MSG1="${ABORTED_MSG1} $(basename "${FILE_TO_UPLOAD}") was aborted." +ABORTED_FIELDS="${FILE_TYPE}\t${FILE_TO_UPLOAD}" +ABORTED_MSG2="uploads" +if ! one_instance --sleep "${SLEEP}" --max-checks "${MAX_CHECKS}" \ + --process-name "${ME}" --pid-file "${PID_FILE}" \ + --aborted-count-file "${ALLSKY_ABORTEDUPLOADS}" --aborted-fields "${ABORTED_FIELDS}" \ + --aborted-msg1 "${ABORTED_MSG1}" --aborted-msg2 "${ABORTED_MSG2}" ; then + exit 1 fi -while : ; do - [[ ! -f ${PID_FILE} ]] && break - - PID=$( < "${PID_FILE}" ) - # shellcheck disable=SC2009 - if ! ps -p "${PID}" | grep --silent "${ME}" ; then - break # Not sure why the PID file existed if the process didn't exist. - fi - - if [[ $NUM_CHECKS -eq ${MAX_CHECKS} ]]; then - echo -en "${YELLOW}" >&2 - echo -n "${ME}: WARNING: Another '${FILE_TYPE}' upload is in" >&2 - echo " progress so the new upload of $(basename "${FILE_TO_UPLOAD}") was aborted." >&2 - echo -n "Made ${NUM_CHECKS} attempts at waiting." >&2 - echo -n " If this happens often, check your network and delay settings." >&2 - echo -e "${NC}" >&2 - ps -fp "${PID}" >&2 - - # Keep track of aborts so user can be notified. - # If it's happening often let the user know. - echo -e "$(date)\t${FILE_TYPE}\t${FILE_TO_UPLOAD}" >> "${ALLSKY_ABORTEDUPLOADS}" - NUM=$( wc -l < "${ALLSKY_ABORTEDUPLOADS}" ) - if [[ ${NUM} -eq 3 || ${NUM} -eq 10 ]]; then - MSG="${NUM} uploads have been aborted waiting for other uploads to finish." - MSG="${MSG}\nThis could be caused by a slow network or other network issues." - if [[ ${NUM} -eq 3 ]]; then - SEVERITY="info" - else - SEVERITY="warning" - MSG="${MSG}\nOnce you have resolved the cause, reset the aborted counter:" - MSG="${MSG}\n    rm -f '${ALLSKY_ABORTEDUPLOADS}'" - fi - "${ALLSKY_SCRIPTS}/addMessage.sh" "${SEVERITY}" "${MSG}" - fi - - exit 2 - else - sleep "${SLEEP_TIME}" - fi - ((NUM_CHECKS++)) -done -echo $$ > "${PID_FILE}" || exit 1 # Convert to lowercase so we don't care if user specified upper or lowercase. PROTOCOL="${PROTOCOL,,}" diff --git a/variables.sh b/variables.sh index aac337c62..abafab247 100644 --- a/variables.sh +++ b/variables.sh @@ -82,9 +82,10 @@ if [[ -z "${ALLSKY_VARIABLE_SET}" ]]; then ALLSKY_INSTALLATION_LOGS="${ALLSKY_CONFIG}/installation_logs" POST_INSTALLATION_ACTIONS="${ALLSKY_INSTALLATION_LOGS}/post-installation_actions.txt" - # Holds temporary list of aborted uploads and timelapse since another one was in progress + # Holds temporary list of aborted processes since another one was in progress. ALLSKY_ABORTEDUPLOADS="${ALLSKY_TMP}/aborted_uploads.txt" ALLSKY_ABORTEDTIMELAPSE="${ALLSKY_TMP}/aborted_timelapse.txt" + ALLSKY_ABORTEDSAVEIMAGE="${ALLSKY_TMP}/aborted_saveImage.txt" # Holds all the dark frames. ALLSKY_DARKS="${ALLSKY_HOME}/darks"