diff --git a/charts/snapshotEngine/templates/configmap.yaml b/charts/snapshotEngine/templates/configmap.yaml index 2b58dd9bf..b0476f6dc 100644 --- a/charts/snapshotEngine/templates/configmap.yaml +++ b/charts/snapshotEngine/templates/configmap.yaml @@ -12,6 +12,7 @@ data: ALL_SUBDOMAINS: {{ $.Values.allSubdomains }} ARCHIVE_SLEEP_DELAY: {{ $.Values.artifactDelay.archive }} ROLLING_SLEEP_DELAY: {{ $.Values.artifactDelay.rolling }} + SCHEMA_URL: {{ $.Values.schemaUrl }} kind: ConfigMap metadata: name: snapshot-configmap diff --git a/charts/snapshotEngine/values.yaml b/charts/snapshotEngine/values.yaml index 4b27e7986..58c6ede0f 100755 --- a/charts/snapshotEngine/values.yaml +++ b/charts/snapshotEngine/values.yaml @@ -96,3 +96,6 @@ allSubdomains: "" artifactDelay: rolling: 0m archive: 0m + +# URL to schema.json file to validate generated metadata against +schemaUrl: "https://oxheadalpha.com/tezos-snapshot-metadata.schema.1.0.json" diff --git a/snapshotEngine/Dockerfile b/snapshotEngine/Dockerfile index 1b31b6227..17c2a1dc4 100644 --- a/snapshotEngine/Dockerfile +++ b/snapshotEngine/Dockerfile @@ -43,10 +43,10 @@ RUN apk --no-cache add \ && apk --no-cache del \ binutils \ && rm -rf /var/cache/apk/* \ - && apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python \ + && apk add --update --no-cache python3-dev && ln -sf python3 /usr/bin/python \ && python3 -m ensurepip \ && pip3 install --no-cache-dir --upgrade pip && \ - pip3 install --no-cache-dir setuptools boto3 datefinder datetime pytz + pip3 install --no-cache-dir setuptools boto3 datefinder datetime pytz jsonschema RUN chown jekyll:jekyll -R /usr/gem diff --git a/snapshotEngine/getAllSnapshotMetadata.py b/snapshotEngine/getAllSnapshotMetadata.py index 70ec6db8f..08689d0cf 100644 --- a/snapshotEngine/getAllSnapshotMetadata.py +++ b/snapshotEngine/getAllSnapshotMetadata.py @@ -1,16 +1,20 @@ from genericpath import exists import os import urllib, json -import urllib.request +from jsonschema import validate +from datetime import datetime +schemaURL = os.environ["SCHEMA_URL"] allSubDomains = os.environ["ALL_SUBDOMAINS"].split(",") snapshotWebsiteBaseDomain = os.environ["SNAPSHOT_WEBSITE_DOMAIN_NAME"] - filename = "tezos-snapshots.json" # Write empty top-level array to initialize json -json_object = [] +artifact_metadata = [] + +urllib.request.urlretrieve(schemaURL, "schema.json") +print("Assembling global metadata file for all subdomains:") print(allSubDomains) # Get each subdomain's base.json and combine all artifacts into 1 metadata file @@ -22,11 +26,36 @@ with urllib.request.urlopen(baseJsonUrl) as url: data = json.loads(url.read().decode()) for entry in data: - json_object.append(entry) + artifact_metadata.append(entry) except urllib.error.HTTPError: continue +now = datetime.now() + +# Matches octez block_timestamp. +# Is ISO 8601 with military offset of Z +dt_string = now.strftime('%Y-%m-%dT%H:%M:%SZ') + +# Meta document that includes the list of storage artifacts among some other useful keys. +metadata_document = json.dumps({ + "date_generated": dt_string, + "org": "Oxhead Alpha", + "$schema": schemaURL, + "data": artifact_metadata, +}, indent=4) + +with open("schema.json","r") as f: + schema = f.read() + +# json.validate() returns None if successful +if not validate(metadata_document, json.loads(schema)): + print("Metadata sucessfully validated against schema!") +else: + raise Exception("Metadata NOT validated against schema!") + + # Write to file with open(filename, "w") as json_file: - json_string = json.dumps(json_object, indent=4) - json_file.write(json_string) + json_file.write(metadata_document) + +print(f"Done assembling global metadata file {filename}") diff --git a/snapshotEngine/getLatestSnapshotMetadata.py b/snapshotEngine/getLatestSnapshotMetadata.py index 532385ebd..6c124139a 100644 --- a/snapshotEngine/getLatestSnapshotMetadata.py +++ b/snapshotEngine/getLatestSnapshotMetadata.py @@ -1,16 +1,12 @@ -import datetime import json -import urllib -import urllib.request from pathlib import Path -import pprint -from datetime import datetime - -import datefinder -import pytz +import random from genericpath import exists -filename = "tezos-snapshots.json" +import pprint +pp = pprint.PrettyPrinter(indent=4) + +filename='tezos-snapshots.json' if exists(filename): print("SUCCESS tezos-snapshots.json exists locally!") @@ -29,7 +25,7 @@ all_snapshots = [{"name": "example", "all_snapshots": {}}] -for snapshot in snapshots: +for snapshot in snapshots['data']: network = snapshot["chain_name"] if network not in snapshots_per_network: snapshots_per_network[network] = [] @@ -39,53 +35,28 @@ network_latest_snapshots = {} network_snapshots = {} - # Initialize date to now, and then update with build date as we iterate - last_tezos_build_datetime = datetime.now().replace(tzinfo=pytz.UTC) - for type, mode, path in [ - ("tarball", "rolling", "rolling-tarball"), - ("tarball", "archive", "archive-tarball"), - ("tezos-snapshot", "rolling", "rolling"), - ]: - # Parses date from tezos build of each artifact, compares to last date, updates if older, otherwise its newer - for snapshot in snapshots: - matches = datefinder.find_dates(snapshot["tezos_version"]) - tezos_build_datetime = list(matches)[0] - if tezos_build_datetime < last_tezos_build_datetime: - latest_tezos_build_version = [ - src - for time, src in datefinder.find_dates( - snapshot["tezos_version"], source=True - ) - ][1] - last_tezos_build_datetime = tezos_build_datetime - - # Snapshots of type (tarball/snapshot) and history mode - typed_snapshots = [ - s - for s in snapshots - if s["artifact_type"] == type and s["history_mode"] == mode - ] - typed_snapshots.sort(key=lambda x: int(x["block_height"]), reverse=True) - - try: - # Keep list of all snapshots - network_snapshots[path] = typed_snapshots - except IndexError: + # Find a lowest version available for a given network, artifact_type, and history_mode + for (artifact_type, history_mode, path) in [("tarball", "rolling", "rolling-tarball"), ("tarball", "archive", "archive-tarball"), ("tezos-snapshot", "rolling", "rolling")]: + # List of snapshot metadata for this particular artifact type and history mode + typed_snapshots = [s for s in snapshots if s["artifact_type"] == artifact_type and s["history_mode"] == history_mode] + + # Lowest version is the top item (int) of a sorted unique list of all the versions for this particular artifact type and history mode + octez_versions = sorted(list(set([ s['tezos_version']['version']['major'] for s in typed_snapshots if 'version' in s['tezos_version'] ]))) + if octez_versions: + lowest_octez_version = octez_versions[0] + else: + # no metadata yet for this namespace, ignoring continue - # Latest should only show oldest supported build so let's filter by the oldest supported version we found above - typed_snapshots = [ - t - for t in typed_snapshots - if latest_tezos_build_version in t["tezos_version"] - ] + network_snapshots[path] = typed_snapshots - try: - # Latest snapshot of type is the first item in typed_snapshots which we just filtered by the latest supported tezos build - network_latest_snapshots[path] = typed_snapshots[0] - except IndexError: - continue + # Latest offered should only show oldest supported build so let's filter by the oldest supported version we found above + typed_snapshots = [d for d in typed_snapshots if 'version' in d['tezos_version'] and d['tezos_version']['version']['major'] == lowest_octez_version ] + # Latest snapshot of type is the last item in typed_snapshots which we just filtered by the latest supported tezos build + network_latest_snapshots[path] = typed_snapshots[-1] + + # This becomes the list of snapshots latest_snapshots.append( { "name": network, @@ -102,9 +73,7 @@ ) Path("_data").mkdir(parents=True, exist_ok=True) -with open(f"_data/snapshot_jekyll_data.json", "w") as f: - json.dump( - {"latest_snapshots": latest_snapshots, "all_snapshots": all_snapshots}, - f, - indent=2, - ) +filename = "_data/snapshot_jekyll_data.json" +with open(filename, 'w') as f: + json.dump({"latest_snapshots": latest_snapshots, "all_snapshots": all_snapshots}, f, indent=2) +print(f"Done writing structured list of snapshots for Jekyll to render webpage: {filename}") diff --git a/snapshotEngine/mainJob.yaml b/snapshotEngine/mainJob.yaml index 98a14484e..586908f8b 100644 --- a/snapshotEngine/mainJob.yaml +++ b/snapshotEngine/mainJob.yaml @@ -34,7 +34,7 @@ spec: # Set up config for headless RPC using new restored storage octez-node config init \ --config-file /home/tezos/.tezos-node/config.json \ - --network "${NETWORK}" \ + --network "${CHAIN_NAME}" \ --data-dir /var/tezos/node/data # Run headless tezos node to validate storage on restored volume @@ -117,16 +117,20 @@ spec: fi # Get BLOCK_TIMESTAMP from RPC - wget -qO- http://localhost:8732/chains/main/blocks/head/header | sed -E 's/.*"timestamp":"?([^,"]*)"?.*/\1/' > /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_TIMESTAMP + wget -qO- http://localhost:8732/chains/main/blocks/head/header | sed -E 's/.*"timestamp":"?([^,"]*)"?.*/\1/' > /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_TIMESTAMP - # Get Tezos Version from octez-node command + # Old version string /usr/local/bin/octez-node --version > /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_VERSION + # Get new version object from RPC + wget -qO- http://localhost:8732/version > /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_RPC_VERSION_INFO + # Print variables for debug printf "%s BLOCK_HASH is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HASH))\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" printf "%s BLOCK_HEIGHT is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HEIGHT)\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" printf "%s BLOCK_TIMESTAMP is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_TIMESTAMP)\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" printf "%s TEZOS_VERSION is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_VERSION)\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + printf "%s TEZOS_RPC_VERSION_INFO is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_RPC_VERSION_INFO)\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" # Blow open permissions for next job to write to volume sudo chmod -R 755 /"${HISTORY_MODE}"-snapshot-cache-volume @@ -142,6 +146,8 @@ spec: envFrom: - configMapRef: name: snapshot-configmap + - configMapRef: + name: tezos-config containers: - name: create-tezos-rolling-snapshot image: "" @@ -164,7 +170,7 @@ spec: octez-node config init \ --config-file /home/tezos/.tezos-node/config.json \ - --network "${NETWORK}" \ + --network "${CHAIN_NAME}" \ --data-dir /var/tezos/node/data if [ "${HISTORY_MODE}" = rolling ]; then @@ -180,11 +186,18 @@ spec: octez-node snapshot import \ /"${HISTORY_MODE}"-snapshot-cache-volume/"${ROLLING_SNAPSHOT_NAME}".rolling \ - --in-memory \ --block "${BLOCK_HASH}" \ --config-file /home/tezos/.tezos-node/config.json \ --data-dir /rolling-tarball-restore/var/tezos/node/data + # Version check for v15 vs v16 conditional + TEZOS_VERSION=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_VERSION) + + # Get context elements and octez snapshot version + if [[ "$TEZOS_VERSION" != '763259c5 (2022-12-01 10:20:58 +0000) (15.1)' ]]; then + /usr/local/bin/octez-node snapshot info --json /"${HISTORY_MODE}"-snapshot-cache-volume/"${ROLLING_SNAPSHOT_NAME}".rolling --json > /"${HISTORY_MODE}"-snapshot-cache-volume/SNAPSHOT_HEADER + fi + rm /rolling-tarball-restore/snapshot-import-in-progress else printf "%s Skipping rolling snapshot import since this job is for an archive node.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" @@ -199,16 +212,11 @@ spec: env: - name: HISTORY_MODE value: "" - - name: NAMESPACE - valueFrom: - configMapKeyRef: - name: snapshot-configmap - key: NAMESPACE - - name: NETWORK_OVERRIDE - valueFrom: - configMapKeyRef: - name: snapshot-configmap - key: NETWORK_OVERRIDE + envFrom: + - configMapRef: + name: snapshot-configmap + - configMapRef: + name: tezos-config - name: zip-and-upload image: "" imagePullPolicy: Always diff --git a/snapshotEngine/zip-and-upload.sh b/snapshotEngine/zip-and-upload.sh index 06a5ba849..0cc40481b 100755 --- a/snapshotEngine/zip-and-upload.sh +++ b/snapshotEngine/zip-and-upload.sh @@ -3,9 +3,14 @@ BLOCK_HEIGHT=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HEIGHT) BLOCK_HASH=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HASH) BLOCK_TIMESTAMP=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_TIMESTAMP) -TEZOS_VERSION=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_VERSION) +#TEZOS_VERSION=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_VERSION) NETWORK="${NAMESPACE%%-*}" export S3_BUCKET="${NAMESPACE%-*}.${SNAPSHOT_WEBSITE_DOMAIN_NAME}" +TEZOS_RPC_VERSION_INFO="$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/TEZOS_RPC_VERSION_INFO)" + +TEZOS_VERSION="$(echo "${TEZOS_RPC_VERSION_INFO}" | jq -r .version)" +TEZOS_VERSION_COMMIT_HASH="$(echo "${TEZOS_RPC_VERSION_INFO}" | jq -r .commit_info.commit_hash)" +TEZOS_VERSION_COMMIT_DATE="$(echo "${TEZOS_RPC_VERSION_INFO}" | jq -r .commit_info.commit_date)" cd / @@ -72,38 +77,54 @@ if [ "${HISTORY_MODE}" = archive ]; then --arg SHA256 "${SHA256}" \ --arg FILESIZE_BYTES "${FILESIZE_BYTES}" \ --arg FILESIZE "${FILESIZE}" \ - --arg TEZOS_VERSION "${TEZOS_VERSION}" \ --arg NETWORK "${NETWORK}" \ --arg HISTORY_MODE "archive" \ --arg ARTIFACT_TYPE "tarball" \ + --arg TEZOS_VERSION_COMMIT_HASH "${TEZOS_VERSION_COMMIT_HASH}" \ + --arg TEZOS_VERSION_COMMIT_DATE "${TEZOS_VERSION_COMMIT_DATE}" \ '{ - "block_hash": $BLOCK_HASH, - "block_height": $BLOCK_HEIGHT, + "block_hash": $BLOCK_HASH, + "block_height": ($BLOCK_HEIGHT|fromjson), "block_timestamp": $BLOCK_TIMESTAMP, "filename": $ARCHIVE_TARBALL_FILENAME, - "sha256": $SHA256, "url": $URL, - "filesize_bytes": $FILESIZE_BYTES, - "filesize": $FILESIZE, - "tezos_version": $TEZOS_VERSION, + "sha256": $SHA256, + "filesize_bytes": ($FILESIZE_BYTES|fromjson), + "filesize": $FILESIZE, "chain_name": $NETWORK, "history_mode": $HISTORY_MODE, - "artifact_type": $ARTIFACT_TYPE + "artifact_type": $ARTIFACT_TYPE, + "tezos_version": { + "implementation": "octez", + "version": "", + "commit_info": { + "commit_hash": $TEZOS_VERSION_COMMIT_HASH, + "commit_date": $TEZOS_VERSION_COMMIT_DATE + } + } }' \ > "${ARCHIVE_TARBALL_FILENAME}".json + # Since version.additional_info will either be another object or "release" we just overwrite it from whatever we got above + # JQ has trouble inserting a key into a file this is the way we opted to insert it + tmp=$(mktemp) + jq --arg version "$TEZOS_VERSION" '.tezos_version.version = ($version|fromjson)' "${ARCHIVE_TARBALL_FILENAME}".json > "$tmp" && mv "$tmp" "${ARCHIVE_TARBALL_FILENAME}".json + # Check metadata json exists - if [ -f "${ARCHIVE_TARBALL_FILENAME}".json ]; then + if [[ -s "${ARCHIVE_TARBALL_FILENAME}".json ]]; then printf "%s Archive Tarball : ${ARCHIVE_TARBALL_FILENAME}.json created.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" - else - printf "%s Archive Tarball : Error creating ${ARCHIVE_TARBALL_FILENAME}.json.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" - fi - # Upload archive tarball metadata json - if ! aws s3 cp "${ARCHIVE_TARBALL_FILENAME}".json s3://"${S3_BUCKET}"/"${ARCHIVE_TARBALL_FILENAME}".json; then - printf "%s Archive Tarball : Error uploading ${ARCHIVE_TARBALL_FILENAME}.json to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + # Optional schema validation + validate_metadata "${ARCHIVE_TARBALL_FILENAME}".json + + # Upload archive tarball metadata json + if ! aws s3 cp "${ARCHIVE_TARBALL_FILENAME}".json s3://"${S3_BUCKET}"/"${ARCHIVE_TARBALL_FILENAME}".json; then + printf "%s Archive Tarball : Error uploading ${ARCHIVE_TARBALL_FILENAME}.json to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + else + printf "%s Archive Tarball : Artifact JSON ${ARCHIVE_TARBALL_FILENAME}.json uploaded to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + fi else - printf "%s Archive Tarball : Artifact JSON ${ARCHIVE_TARBALL_FILENAME}.json uploaded to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + printf "%s Archive Tarball : Error creating ${ARCHIVE_TARBALL_FILENAME}.json.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" fi # Create archive tarball redirect file @@ -236,37 +257,54 @@ if [ "${HISTORY_MODE}" = rolling ]; then --arg SHA256 "$SHA256" \ --arg FILESIZE_BYTES "$FILESIZE_BYTES" \ --arg FILESIZE "$FILESIZE" \ - --arg TEZOS_VERSION "$TEZOS_VERSION" \ --arg NETWORK "$NETWORK" \ --arg HISTORY_MODE "rolling" \ --arg ARTIFACT_TYPE "tarball" \ + --arg TEZOS_VERSION_COMMIT_HASH "${TEZOS_VERSION_COMMIT_HASH}" \ + --arg TEZOS_VERSION_COMMIT_DATE "${TEZOS_VERSION_COMMIT_DATE}" \ '{ - "block_hash": $BLOCK_HASH, - "block_height": $BLOCK_HEIGHT, + "block_hash": $BLOCK_HASH, + "block_height": ($BLOCK_HEIGHT|fromjson), "block_timestamp": $BLOCK_TIMESTAMP, "filename": $ROLLING_TARBALL_FILENAME, "url": $URL, "sha256": $SHA256, - "filesize_bytes": $FILESIZE_BYTES, - "filesize": $FILESIZE, - "tezos_version": $TEZOS_VERSION, + "filesize_bytes": ($FILESIZE_BYTES|fromjson), + "filesize": $FILESIZE, "chain_name": $NETWORK, "history_mode": $HISTORY_MODE, - "artifact_type": $ARTIFACT_TYPE + "artifact_type": $ARTIFACT_TYPE, + "tezos_version": { + "implementation": "octez", + "version": "", + "commit_info": { + "commit_hash": $TEZOS_VERSION_COMMIT_HASH, + "commit_date": $TEZOS_VERSION_COMMIT_DATE + } + } }' \ > "${ROLLING_TARBALL_FILENAME}".json + + # Since version.additional_info will either be another object or "release" we just overwrite it from whatever we got above + # JQ has trouble inserting a key into a file this is the way we opted to insert it + tmp=$(mktemp) + jq --arg version "$TEZOS_VERSION" '.tezos_version.version = ($version|fromjson)' "${ROLLING_TARBALL_FILENAME}".json > "$tmp" && mv "$tmp" "${ROLLING_TARBALL_FILENAME}".json - if [ -f "${ROLLING_TARBALL_FILENAME}".json ]; then + # Check metadata exists + if [[ -s "${ROLLING_TARBALL_FILENAME}".json ]]; then printf "%s Rolling Tarball : ${ROLLING_TARBALL_FILENAME}.json created.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" - else - printf "%s Rolling Tarball : Error creating ${ROLLING_TARBALL_FILENAME}.json locally.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" - fi - # upload metadata json - if ! aws s3 cp "${ROLLING_TARBALL_FILENAME}".json s3://"${S3_BUCKET}"/"${ROLLING_TARBALL_FILENAME}".json; then - printf "%s Rolling Tarball : Error uploading ${ROLLING_TARBALL_FILENAME}.json to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + # Optional schema validation + validate_metadata "${ROLLING_TARBALL_FILENAME}".json + + # upload metadata json + if ! aws s3 cp "${ROLLING_TARBALL_FILENAME}".json s3://"${S3_BUCKET}"/"${ROLLING_TARBALL_FILENAME}".json; then + printf "%s Rolling Tarball : Error uploading ${ROLLING_TARBALL_FILENAME}.json to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + else + printf "%s Rolling Tarball : Metadata JSON ${ROLLING_TARBALL_FILENAME}.json uploaded to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + fi else - printf "%s Rolling Tarball : Metadata JSON ${ROLLING_TARBALL_FILENAME}.json uploaded to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + printf "%s Rolling Tarball : Error creating ${ROLLING_TARBALL_FILENAME}.json locally.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" fi # Tarball redirect file @@ -318,6 +356,15 @@ if [ "${HISTORY_MODE}" = rolling ]; then FILESIZE=$(echo "${FILESIZE_BYTES}" | awk '{ suffix="KMGT"; for(i=0; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }' | xargs ) SHA256=$(sha256sum "${ROLLING_SNAPSHOT}" | awk '{print $1}') + + TEZOS_VERSION_MAJOR="$(echo "${TEZOS_RPC_VERSION_INFO}" | jq .version.major)" + + if [[ $TEZOS_VERSION_MAJOR -lt 16 ]]; then + SNAPSHOT_VERSION=4 + else + SNAPSHOT_HEADER=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/SNAPSHOT_HEADER) + SNAPSHOT_VERSION="$(echo "${SNAPSHOT_HEADER}" | jq .snapshot_header.version)" + fi jq -n \ --arg BLOCK_HASH "$BLOCK_HASH" \ @@ -328,31 +375,57 @@ if [ "${HISTORY_MODE}" = rolling ]; then --arg SHA256 "$SHA256" \ --arg FILESIZE_BYTES "$FILESIZE_BYTES" \ --arg FILESIZE "$FILESIZE" \ - --arg TEZOS_VERSION "$TEZOS_VERSION" \ --arg NETWORK "$NETWORK" \ --arg HISTORY_MODE "rolling" \ --arg ARTIFACT_TYPE "tezos-snapshot" \ + --arg TEZOS_VERSION_COMMIT_HASH "${TEZOS_VERSION_COMMIT_HASH}" \ + --arg TEZOS_VERSION_COMMIT_DATE "${TEZOS_VERSION_COMMIT_DATE}" \ + --arg SNAPSHOT_VERSION "$SNAPSHOT_VERSION" \ '{ - "block_hash": $BLOCK_HASH, - "block_height": $BLOCK_HEIGHT, + "block_hash": $BLOCK_HASH, + "block_height": ($BLOCK_HEIGHT|fromjson), "block_timestamp": $BLOCK_TIMESTAMP, "filename": $ROLLING_SNAPSHOT_FILENAME, "url": $URL, - "filesize_bytes": $FILESIZE_BYTES, - "filesize": $FILESIZE, "sha256": $SHA256, - "tezos_version": $TEZOS_VERSION, + "filesize_bytes": ($FILESIZE_BYTES|fromjson), + "filesize": $FILESIZE, "chain_name": $NETWORK, "history_mode": $HISTORY_MODE, - "artifact_type": $ARTIFACT_TYPE + "artifact_type": $ARTIFACT_TYPE, + "tezos_version":{ + "implementation": "octez", + "version": "", + "commit_info": { + "commit_hash": $TEZOS_VERSION_COMMIT_HASH, + "commit_date": $TEZOS_VERSION_COMMIT_DATE + } + }, + "snapshot_version": ($SNAPSHOT_VERSION|fromjson), }' \ > "${ROLLING_SNAPSHOT_FILENAME}".json - - printf "%s Rolling Tezos : Metadata JSON created.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" - # upload metadata json - aws s3 cp "${ROLLING_SNAPSHOT_FILENAME}".json s3://"${S3_BUCKET}"/"${ROLLING_SNAPSHOT_FILENAME}".json - printf "%s Rolling Tezos : Metadata JSON ${ROLLING_SNAPSHOT_FILENAME}.json uploaded to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + # Since version.additional_info will either be another object or "release" we just overwrite it from whatever we got above + # JQ has trouble inserting a key into a file this is the way we opted to insert it + tmp=$(mktemp) + jq --arg version "$TEZOS_VERSION" '.tezos_version.version = ($version|fromjson)' "${ROLLING_SNAPSHOT_FILENAME}".json > "$tmp" && mv "$tmp" "${ROLLING_SNAPSHOT_FILENAME}".json + + # Check metadata json exists + if [[ -s "${ROLLING_SNAPSHOT_FILENAME}".json ]]; then + printf "%s Rolling Snapshot : ${ROLLING_SNAPSHOT_FILENAME}.json created.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + + # Optional schema validation + validate_metadata "${ROLLING_SNAPSHOT_FILENAME}".json + + # Upload Rolling Snapshot metadata json + if ! aws s3 cp "${ROLLING_SNAPSHOT_FILENAME}".json s3://"${S3_BUCKET}"/"${ROLLING_SNAPSHOT_FILENAME}".json; then + printf "%s Rolling Snapshot : Error uploading ${ROLLING_SNAPSHOT_FILENAME}.json to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + else + printf "%s Rolling Snapshot : Artifact JSON ${ROLLING_SNAPSHOT_FILENAME}.json uploaded to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + fi + else + printf "%s Rolling Snapshot : Error creating ${ROLLING_SNAPSHOT_FILENAME}.json.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + fi # Rolling snapshot redirect object touch rolling @@ -431,6 +504,14 @@ fi # build site pages python /getAllSnapshotMetadata.py +# Fail if python raised exception (validation failure) +ret=$? +if [[ "${ret}" -ne 0 ]]; then + printf "%s Metadata did not validate sucessfully. Exiting... \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")" + sleep 20 + exit 1 +fi + # Check if tezos-snapshots.json exists # tezos-snapshots.json is a list of all snapshots in all buckets if [[ ! -f tezos-snapshots.json ]]; then diff --git a/utils/config-generator.py b/utils/config-generator.py index 45cf8ba4b..30462230a 100755 --- a/utils/config-generator.py +++ b/utils/config-generator.py @@ -702,7 +702,7 @@ def create_node_snapshot_config_json(history_mode): # find snapshot matching all the requested fields matching_snapshots = [ s - for s in all_snapshots + for s in all_snapshots['data'] if s.get("history_mode") == history_mode and s.get("artifact_type") == artifact_type and s.get("chain_name") == network_name diff --git a/utils/sidecar.py b/utils/sidecar.py index 633b16168..ab60fb026 100755 --- a/utils/sidecar.py +++ b/utils/sidecar.py @@ -46,7 +46,7 @@ def sync_checker(): % (age_in_secs, AGE_LIMIT_IN_SECS), 500, ) - application.logger.error(err) + print(err) return err return "Chain is bootstrapped"