Skip to content

Commit

Permalink
logic for dynamically calculating sleep time (#608)
Browse files Browse the repository at this point in the history
  • Loading branch information
@orcutt989
orcutt989 authored Oct 3, 2023
1 parent 40bdb62 commit 6b9ac52
Showing 1 changed file with 64 additions and 42 deletions.
106 changes: 64 additions & 42 deletions snapshotEngine/snapshot-maker.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,26 +28,6 @@ getSnapshotNames() {
kubectl get volumesnapshots -o jsonpath="{.items[?(.status.readyToUse==$readyToUse)].metadata.name}" --namespace "$NAMESPACE" "$@"
}

# SLEEP_TIME=0m

# if [ "${HISTORY_MODE}" = "archive" ]; then
# SLEEP_TIME="${ARCHIVE_SLEEP_DELAY}"
# if [ "${ARCHIVE_SLEEP_DELAY}" != "0m" ]; then
# printf "%s artifactDelay.archive is set to %s sleeping...\n" "$(date "+%Y-%m-%d %H:%M:%S")" "${ARCHIVE_SLEEP_DELAY}"
# fi
# elif [ "${HISTORY_MODE}" = "rolling" ]; then
# SLEEP_TIME="${ROLLING_SLEEP_DELAY}"
# if [ "${ROLLING_SLEEP_DELAY}" != "0m" ]; then
# printf "%s artifactDelay.rolling is set to %s sleeping...\n" "$(date "+%Y-%m-%d %H:%M:%S")" "${ROLLING_SLEEP_DELAY}"
# fi
# fi

# if [ "${SLEEP_TIME}" = "0m" ]; then
# printf "%s artifactDelay.HISTORY_MODE was not set! No delay...\n" "$(date "+%Y-%m-%d %H:%M:%S")"
# fi

# sleep "${SLEEP_TIME}"

cd /

ZIP_AND_UPLOAD_JOB_NAME=zip-and-upload-"${HISTORY_MODE}"
Expand Down Expand Up @@ -88,17 +68,79 @@ if [ "$(kubectl get pvc "${HISTORY_MODE}"-snap-volume)" ]; then
sleep 5
fi

# Check latest artifact and sleep if its too new
# This was done because nodes sometimes OOM and jobs are restarted
# Resulting in more artifacts created than should be with
# a given sleep time. IE 3 days sleep should result in 2 artifacts in a 7 day lifecycle policy
# but if the job restarts during its sleeping time it would result in more, multiple per day even.

SLEEP_TIME=0m

if [ "${HISTORY_MODE}" = "archive" ]; then
SLEEP_TIME="${ARCHIVE_SLEEP_DELAY}"
if [ "${ARCHIVE_SLEEP_DELAY}" != "0m" ]; then
printf "%s artifactDelay.archive is set to %s.\n" "$(date "+%Y-%m-%d %H:%M:%S")" "${ARCHIVE_SLEEP_DELAY}"
fi
elif [ "${HISTORY_MODE}" = "rolling" ]; then
SLEEP_TIME="${ROLLING_SLEEP_DELAY}"
if [ "${ROLLING_SLEEP_DELAY}" != "0m" ]; then
printf "%s artifactDelay.rolling is set to %s.\n" "$(date "+%Y-%m-%d %H:%M:%S")" "${ROLLING_SLEEP_DELAY}"
fi
fi

if [ "${SLEEP_TIME}" = "0m" ]; then
printf "%s artifactDelay.HISTORY_MODE was not set! No delay...\n" "$(date "+%Y-%m-%d %H:%M:%S")"
else
# Latest timestamp of this network's artifact of this history mode
LATEST_ARTIFACT_TIMESTAMP=$(curl https://${NAMESPACE}.nyc3.digitaloceanspaces.com/base.json | \
jq --arg HISTORY_MODE "${HISTORY_MODE}" \
'[.[] | select(.history_mode==$HISTORY_MODE)] | sort_by(.block_timestamp) | last | .block_timestamp' | tr -d '"')

# If base.json doesnt exists continue
if [[ -n ${LATEST_ARTIFACT_TIMESTAMP} ]]; then
printf "%s Latest artifact timestamp is %s.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" "${LATEST_ARTIFACT_TIMESTAMP}"
printf "%s Sleep time is %s.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" "${SLEEP_TIME}"

## Now minus artifact timestamp
ARTIFACT_EPOCH_TIME=$(date -d "${LATEST_ARTIFACT_TIMESTAMP}" -D "%Y-%m-%dT%H:%M:%SZ" +%s)

# Check if sleep time is in hours or days and convert to minutes
if [[ -n $(echo $SLEEP_TIME | grep d) ]]; then
# Converting hours to minutes
SLEEP_TIME_MINUTES=$(( ${SLEEP_TIME%?} * 24 * 60 ))
else
# if its hours already just pop off the h
SLEEP_TIME_MINUTES=$(( "${SLEEP_TIME%?}" * 60 ))
fi

# Age of artifact in minutes
ARTIFACT_AGE=$(( ($(date +%s) - ARTIFACT_EPOCH_TIME) / 60 ))

printf "%s Latest artifact is %s minutes old.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" "${ARTIFACT_AGE}"
printf "%s Our set sleep time in minutes is %s.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" "${SLEEP_TIME_MINUTES}"

# If the age is less than our sleep minutes we need to continue to sleep
if [[ ${ARTIFACT_AGE} -lt ${SLEEP_TIME_MINUTES} ]]; then
TIME_LEFT=$(( SLEEP_TIME_MINUTES - ARTIFACT_AGE ))
printf "%s We need to sleep for %s minutes.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" "${TIME_LEFT}"
sleep "${TIME_LEFT}"m
else
printf "%s Newest artifact is older than sleep time starting job!.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
fi
fi
fi

# Take volume snapshot
current_date=$(date "+%Y-%m-%d-%H-%M-%S" "$@")
export SNAPSHOT_NAME="$current_date-$HISTORY_MODE-node-snapshot"
# Update volume snapshot name
yq e -i '.metadata.name=strenv(SNAPSHOT_NAME)' createVolumeSnapshot.yaml

printf "%s Creating snapshot ${SNAPSHOT_NAME} in ${NAMESPACE}.\n" "$(timestamp)"
printf "%s Creating snapshot ${SNAPSHOT_NAME} in ${NAMESPACE}.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"

# Create snapshot
if ! kubectl apply -f createVolumeSnapshot.yaml; then
printf "%s ERROR creating volumeSnapshot ${SNAPSHOT_NAME} in ${NAMESPACE} .\n" "$(timestamp)"
printf "%s ERROR creating volumeSnapshot ${SNAPSHOT_NAME} in ${NAMESPACE} .\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
exit 1
fi

Expand Down Expand Up @@ -330,26 +372,6 @@ if ! [ "$(kubectl get jobs "zip-and-upload-${HISTORY_MODE}" --namespace "${NAMES
# Delete all volumesnapshots so they arent setting around accruing charges
kubectl delete vs -l history_mode=$HISTORY_MODE

SLEEP_TIME=0m

if [ "${HISTORY_MODE}" = "archive" ]; then
SLEEP_TIME="${ARCHIVE_SLEEP_DELAY}"
if [ "${ARCHIVE_SLEEP_DELAY}" != "0m" ]; then
printf "%s artifactDelay.archive is set to %s sleeping...\n" "$(date "+%Y-%m-%d %H:%M:%S")" "${ARCHIVE_SLEEP_DELAY}"
fi
elif [ "${HISTORY_MODE}" = "rolling" ]; then
SLEEP_TIME="${ROLLING_SLEEP_DELAY}"
if [ "${ROLLING_SLEEP_DELAY}" != "0m" ]; then
printf "%s artifactDelay.rolling is set to %s sleeping...\n" "$(date "+%Y-%m-%d %H:%M:%S")" "${ROLLING_SLEEP_DELAY}"
fi
fi

if [ "${SLEEP_TIME}" = "0m" ]; then
printf "%s artifactDelay.HISTORY_MODE was not set! No delay...\n" "$(date "+%Y-%m-%d %H:%M:%S")"
fi

sleep "${SLEEP_TIME}"

fi

printf "%s Deleting temporary snapshot volume.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
Expand Down

0 comments on commit 6b9ac52

Please sign in to comment.