From a54bdbae8a3b0c9111bd549dad5da419bb84fe32 Mon Sep 17 00:00:00 2001
From: orcutt989 <jimmy.orcutt@tqtezos.com>
Date: Fri, 1 Jul 2022 16:23:47 -0400
Subject: [PATCH 1/6] add optional network value to override

---
 charts/snapshotEngine/templates/configmap.yaml |  1 +
 charts/snapshotEngine/values.yaml              |  5 +++++
 snapshotEngine/mainJob.yaml                    | 18 ++++++++++++++----
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/charts/snapshotEngine/templates/configmap.yaml b/charts/snapshotEngine/templates/configmap.yaml
index 927226031..123afc093 100644
--- a/charts/snapshotEngine/templates/configmap.yaml
+++ b/charts/snapshotEngine/templates/configmap.yaml
@@ -10,6 +10,7 @@ data:
   SERVICE_ACCOUNT: {{ $.Values.service_account }}
   RESTORED_STORAGE_INIT_TIME: {{ $.Values.restoredStorageInitTime }}
   DISABLE_STORAGE_INIT_TIME: {{ $.Values.disableStorageInitTime | quote }}
+  NETWORK: {{ $.Values.network }}
 kind: ConfigMap
 metadata:
   name: snapshot-configmap
diff --git a/charts/snapshotEngine/values.yaml b/charts/snapshotEngine/values.yaml
index 7237cc8f4..463744edc 100755
--- a/charts/snapshotEngine/values.yaml
+++ b/charts/snapshotEngine/values.yaml
@@ -86,3 +86,8 @@ restoredStorageInitTime: 2m
 
 # We also provide the ability to disable the time limit for debugging purposes.
 disableStorageInitTime: "false"
+
+# Used to start the snapshotable node, and for the initialization network on the restored snapshot.
+# Can be useful if namespace does not begin with network name.
+# Optional, defaults to beginning word of namespace before "-shots" ex "mainnet-shots" = "mainnet"
+network: ""
\ No newline at end of file
diff --git a/snapshotEngine/mainJob.yaml b/snapshotEngine/mainJob.yaml
index 4f712da05..546429de6 100644
--- a/snapshotEngine/mainJob.yaml
+++ b/snapshotEngine/mainJob.yaml
@@ -28,8 +28,8 @@ spec:
               # Set error trap in order to kill this job with the timer later
               trap "exit" SIGINT SIGTERM
 
-              # Strip network from namespace
-              NETWORK="${NAMESPACE%%-*}"
+              # Strip network from namespace or use configmap value
+              NETWORK="${NETWORK:-${NAMESPACE%%-*}}"
 
               # Set up config for headless RPC using new restored storage
               tezos-node config init \
@@ -154,6 +154,11 @@ spec:
                 configMapKeyRef:
                   name: snapshot-configmap
                   key: NAMESPACE
+            - name: NETWORK
+              valueFrom:
+                configMapKeyRef:
+                  name: snapshot-configmap
+                  key: NETWORK
       containers:
         - name: create-tezos-rolling-snapshot
           image: ""
@@ -167,8 +172,8 @@ spec:
               sudo chown -R 100:100 /"${HISTORY_MODE}"-snapshot-cache-volume
               sudo chown -R 100:100 /rolling-tarball-restore
 
-              # Strip network from namespace
-              NETWORK="${NAMESPACE%%-*}"
+              # Strip network from namespace or use configmap value
+              NETWORK="${NETWORK:-${NAMESPACE%%-*}}"
 
               BLOCK_HEIGHT=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HEIGHT)
               BLOCK_HASH=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HASH)
@@ -215,6 +220,11 @@ spec:
                 configMapKeyRef:
                   name: snapshot-configmap
                   key: NAMESPACE
+            - name: NETWORK
+              valueFrom:
+                configMapKeyRef:
+                  name: snapshot-configmap
+                  key: NETWORK
         - name: zip-and-upload
           image: ""
           imagePullPolicy: Always

From 26e5b84b59d4a90cbe2442502c8dc76d3a58da91 Mon Sep 17 00:00:00 2001
From: orcutt989 <jimmy.orcutt@tqtezos.com>
Date: Fri, 1 Jul 2022 16:27:48 -0400
Subject: [PATCH 2/6] fix rolling snapshot name

---
 snapshotEngine/mainJob.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/snapshotEngine/mainJob.yaml b/snapshotEngine/mainJob.yaml
index 546429de6..4aa3e9b2e 100644
--- a/snapshotEngine/mainJob.yaml
+++ b/snapshotEngine/mainJob.yaml
@@ -177,7 +177,7 @@ spec:
 
               BLOCK_HEIGHT=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HEIGHT)
               BLOCK_HASH=$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HASH)
-              ROLLING_SNAPSHOT_NAME="${NETWORK}"-"${BLOCK_HEIGHT}"
+              ROLLING_SNAPSHOT_NAME="${NAMESPACE%%-*}"-"${BLOCK_HEIGHT}"
 
               tezos-node config init \
               --config-file /home/tezos/.tezos-node/config.json \

From 4c2873aaf6f39b117a18f7a6581c07b213a12c7f Mon Sep 17 00:00:00 2001
From: orcutt989 <jimmy.orcutt@tqtezos.com>
Date: Fri, 1 Jul 2022 17:03:08 -0400
Subject: [PATCH 3/6] costruct base.json out of individual files and flatten
 schema

---
 snapshotEngine/zip-and-upload.sh | 196 ++++---------------------------
 1 file changed, 20 insertions(+), 176 deletions(-)

diff --git a/snapshotEngine/zip-and-upload.sh b/snapshotEngine/zip-and-upload.sh
index d3a6848d7..61058f8fa 100755
--- a/snapshotEngine/zip-and-upload.sh
+++ b/snapshotEngine/zip-and-upload.sh
@@ -17,36 +17,6 @@ printf "%s BLOCK_HASH is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_
 printf "%s BLOCK_HEIGHT is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_HEIGHT)\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
 printf "%s BLOCK_TIMESTAMP is...$(cat /"${HISTORY_MODE}"-snapshot-cache-volume/BLOCK_TIMESTAMP)\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
 
-# Download base.json or create if if it doesn't exist
-if ! aws s3api head-object --bucket "${S3_BUCKET}" --key "base.json" > /dev/null; then
-    printf "%s Check base.json : Did not detect in S3.  Creating base.json locally to append and upload later.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    if ! touch base.json; then
-        printf "%s Create base.json : Error creating file base.json locally. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    else
-        printf "%s Create base.json : Created file base.json. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    fi
-else
-    printf "%s Check base.json : Exists in S3.  Downloading to append new information and will upload later. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    if ! aws s3 cp s3://"${S3_BUCKET}"/base.json base.json > /dev/null; then
-        printf "%s Download base.json : Error downloading file base.json from S3. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    else
-        printf "%s Download base.json : Downloaded file base.json from S3. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    fi
-fi
-
-# Check if base.json exists locally
-if test -f base.json; then
-    printf "%s Check base.json : File base.json exists locally. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    # Write empty array if empty
-    if ! [ -s "base.json" ]
-    then
-    # It is. Write an empty array to it
-    echo '[]' > "base.json"
-    fi
-else
-    printf "%s Check base.json : File base.json does not exist locally. \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-fi
-
 #
 # Archive Tarball
 #
@@ -87,56 +57,6 @@ if [ "${HISTORY_MODE}" = archive ]; then
         --output text)
     FILESIZE=$(echo "${FILESIZE_BYTES}" | awk '{ suffix="KMGT"; for(i=0; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }' | xargs)
 
-    # Add file to base.json
-    # have to do it here because base.json is being overwritten
-    # by other snapshot actions that are faster
-    tmp=$(mktemp)
-    cp base.json "${tmp}"
-
-    if ! jq \
-    --arg BLOCK_HASH "$BLOCK_HASH" \
-    --arg BLOCK_HEIGHT "$BLOCK_HEIGHT" \
-    --arg BLOCK_TIMESTAMP "$BLOCK_TIMESTAMP" \
-    --arg ARCHIVE_TARBALL_FILENAME "$ARCHIVE_TARBALL_FILENAME" \
-    --arg SHA256 "$SHA256" \
-    --arg FILESIZE_BYTES "$FILESIZE_BYTES" \
-    --arg FILESIZE "$FILESIZE" \
-    --arg TEZOS_VERSION "$TEZOS_VERSION" \
-    --arg NETWORK "$NETWORK" \
-    --arg HISTORY_MODE "archive" \
-    --arg ARTIFACT_TYPE "tarball" \
-    '. |= 
-    [
-        {
-                ($ARCHIVE_TARBALL_FILENAME): {
-                "contents": {
-                    "block_hash": $BLOCK_HASH,
-                    "block_height": $BLOCK_HEIGHT,
-                    "block_timestamp": $BLOCK_TIMESTAMP,
-                    "sha256": $SHA256,
-                    "filesize_bytes": $FILESIZE_BYTES,
-                    "filesize": $FILESIZE,
-                    "tezos_version": $TEZOS_VERSION,
-                    "chain_name": $NETWORK,
-                    "history_mode": $HISTORY_MODE,
-                    "artifact_type": $ARTIFACT_TYPE
-                }
-            }
-        }
-    ] 
-    + .' "${tmp}" > base.json && rm "${tmp}";then
-        printf "%s Archive Tarball base.json: Error updating base.json.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    else
-        printf "%s Archive Tarball : Sucessfully updated base.json with artifact information.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    fi
-
-    #Upload base.json
-    if ! aws s3 cp base.json s3://"${S3_BUCKET}"/base.json; then
-        printf "%s Upload base.json : Error uploading file base.json to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    else
-        printf "%s Upload base.json : File base.json sucessfully uploaded to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    fi
-
     # Check if archive-tarball exists in S3 and process redirect
     if ! aws s3api head-object --bucket "${S3_BUCKET}" --key "${ARCHIVE_TARBALL_FILENAME}" > /dev/null; then
         printf "%s Archive Tarball : Error uploading ${ARCHIVE_TARBALL_FILENAME} to S3 Bucket ${S3_BUCKET}.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
@@ -282,54 +202,6 @@ if [ "${HISTORY_MODE}" = rolling ]; then
     --output text)
     FILESIZE=$(echo "${FILESIZE_BYTES}" | awk '{ suffix="KMGT"; for(i=0; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }' | xargs)
 
-    # Add file to base.json
-    tmp=$(mktemp)
-    cp base.json "${tmp}"
-
-    if ! jq \
-    --arg BLOCK_HASH "$BLOCK_HASH" \
-    --arg BLOCK_HEIGHT "$BLOCK_HEIGHT" \
-    --arg BLOCK_TIMESTAMP "$BLOCK_TIMESTAMP" \
-    --arg ROLLING_TARBALL_FILENAME "$ROLLING_TARBALL_FILENAME" \
-    --arg SHA256 "$SHA256" \
-    --arg FILESIZE_BYTES "$FILESIZE_BYTES" \
-    --arg FILESIZE "$FILESIZE" \
-    --arg TEZOS_VERSION "$TEZOS_VERSION" \
-    --arg NETWORK "$NETWORK" \
-    --arg HISTORY_MODE "rolling" \
-    --arg ARTIFACT_TYPE "tarball" \
-    '. |= 
-    [
-        {
-                ($ROLLING_TARBALL_FILENAME): {
-                "contents": {
-                    "block_hash": $BLOCK_HASH,
-                    "block_height": $BLOCK_HEIGHT,
-                    "block_timestamp": $BLOCK_TIMESTAMP,
-                    "sha256": $SHA256,
-                    "filesize_bytes": $FILESIZE_BYTES,
-                    "filesize": $FILESIZE,
-                    "tezos_version": $TEZOS_VERSION,
-                    "chain_name": $NETWORK,
-                    "history_mode": $HISTORY_MODE,
-                    "artifact_type": $ARTIFACT_TYPE
-                }
-            }
-        }
-    ] 
-    + .' "${tmp}" > base.json && rm "${tmp}";then
-        printf "%s Rolling Tarball base.json: Error updating base.json.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    else
-        printf "%s Rolling Tarball : Sucessfully updated base.json with artifact information.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    fi
-
-    #Upload base.json
-    if ! aws s3 cp base.json s3://"${S3_BUCKET}"/base.json; then
-        printf "%s Upload base.json : Error uploading file base.json to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    else
-        printf "%s Upload base.json : File base.json sucessfully uploaded to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-    fi
-
     # Check if rolling-tarball exists and process redirect
     if ! aws s3api head-object --bucket "${S3_BUCKET}" --key "${ROLLING_TARBALL_FILENAME}" > /dev/null; then
         printf "%s Rolling Tarball : Error uploading ${ROLLING_TARBALL_FILENAME} to S3 Bucket ${S3_BUCKET}.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
@@ -428,54 +300,6 @@ if [ "${HISTORY_MODE}" = rolling ]; then
             FILESIZE=$(echo "${FILESIZE_BYTES}" | awk '{ suffix="KMGT"; for(i=0; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }' | xargs )
             SHA256=$(sha256sum "${ROLLING_SNAPSHOT}" | awk '{print $1}')
 
-            # Add file to base.json
-            tmp=$(mktemp)
-            cp base.json "${tmp}"
-
-            if ! jq \
-            --arg BLOCK_HASH "$BLOCK_HASH" \
-            --arg BLOCK_HEIGHT "$BLOCK_HEIGHT" \
-            --arg BLOCK_TIMESTAMP "$BLOCK_TIMESTAMP" \
-            --arg ROLLING_SNAPSHOT_FILENAME "$ROLLING_SNAPSHOT_FILENAME" \
-            --arg SHA256 "$SHA256" \
-            --arg FILESIZE_BYTES "$FILESIZE_BYTES" \
-            --arg FILESIZE "$FILESIZE" \
-            --arg TEZOS_VERSION "$TEZOS_VERSION" \
-            --arg NETWORK "$NETWORK" \
-            --arg HISTORY_MODE "rolling" \
-            --arg ARTIFACT_TYPE "tezos-snapshot" \
-            '. |= 
-            [
-                {
-                        ($ROLLING_SNAPSHOT_FILENAME): {
-                        "contents": {
-                            "block_hash": $BLOCK_HASH,
-                            "block_height": $BLOCK_HEIGHT,
-                            "block_timestamp": $BLOCK_TIMESTAMP,
-                            "sha256": $SHA256,
-                            "filesize_bytes": $FILESIZE_BYTES,
-                            "filesize": $FILESIZE,
-                            "tezos_version": $TEZOS_VERSION,
-                            "chain_name": $NETWORK,
-                            "history_mode": $HISTORY_MODE,
-                            "artifact_type": $ARTIFACT_TYPE
-                        }
-                    }
-                }
-            ] 
-            + .' "${tmp}" > base.json && rm "${tmp}";then
-                printf "%s Rolling Snapshot base.json: Error updating base.json.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-            else
-                printf "%s Rolling Snapshot : Sucessfully updated base.json with artifact information.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-            fi
-
-            #Upload base.json
-            if ! aws s3 cp base.json s3://"${S3_BUCKET}"/base.json; then
-                printf "%s Upload base.json : Error uploading file base.json to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-            else
-                printf "%s Upload base.json : File base.json sucessfully uploaded to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-            fi
-
             jq -n \
             --arg BLOCK_HASH "$BLOCK_HASH" \
             --arg BLOCK_HEIGHT "$BLOCK_HEIGHT" \
@@ -586,3 +410,23 @@ if ! aws s3 cp _site/ s3://"${S3_BUCKET}" --recursive --include "*"; then
 else
     printf "%s Website Build & Deploy  : Sucessfully uploaded website to S3.\n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
 fi
+
+# Build base.json from existing metadata files
+
+# Create new base.json locally
+touch base.json
+echo '[]' > "base.json"
+
+tmp=$(mktemp)
+cp base.json "${tmp}"
+
+aws s3 ls s3://ithacanet.xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | while read ITEM; do
+    jq --argjson file "$(curl https://$NETWORK.xtz-shots.io/$ITEM)" '. += [$file]' "${tmp}" > base.json && rm "${tmp}"
+done
+
+#Upload base.json
+if ! aws s3 cp base.json s3://"${S3_BUCKET}"/base.json; then
+    printf "%s Upload base.json : Error uploading file base.json to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
+else
+    printf "%s Upload base.json : File base.json sucessfully uploaded to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
+fi
\ No newline at end of file

From e8653d413f16e215ace9d7c0e6c40ed3435aed1e Mon Sep 17 00:00:00 2001
From: orcutt989 <jimmy.orcutt@tqtezos.com>
Date: Tue, 5 Jul 2022 11:39:34 -0400
Subject: [PATCH 4/6] fix network name

---
 snapshotEngine/mainJob.yaml      | 1 +
 snapshotEngine/zip-and-upload.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/snapshotEngine/mainJob.yaml b/snapshotEngine/mainJob.yaml
index 4aa3e9b2e..d0a1e000e 100644
--- a/snapshotEngine/mainJob.yaml
+++ b/snapshotEngine/mainJob.yaml
@@ -197,6 +197,7 @@ spec:
 
                 tezos-node snapshot import \
                 /"${HISTORY_MODE}"-snapshot-cache-volume/"${ROLLING_SNAPSHOT_NAME}".rolling \
+                --in-memory
                 --block "${BLOCK_HASH}" \
                 --config-file /home/tezos/.tezos-node/config.json \
                 --data-dir /rolling-tarball-restore/var/tezos/node/data
diff --git a/snapshotEngine/zip-and-upload.sh b/snapshotEngine/zip-and-upload.sh
index 61058f8fa..f51c4fd56 100755
--- a/snapshotEngine/zip-and-upload.sh
+++ b/snapshotEngine/zip-and-upload.sh
@@ -420,7 +420,7 @@ echo '[]' > "base.json"
 tmp=$(mktemp)
 cp base.json "${tmp}"
 
-aws s3 ls s3://ithacanet.xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | while read ITEM; do
+aws s3 ls s3://"$NETWORK".xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | while read ITEM; do
     jq --argjson file "$(curl https://$NETWORK.xtz-shots.io/$ITEM)" '. += [$file]' "${tmp}" > base.json && rm "${tmp}"
 done
 

From b377004211f3dd5ae4016cc02825c85e6b756516 Mon Sep 17 00:00:00 2001
From: orcutt989 <jimmy.orcutt@tqtezos.com>
Date: Tue, 5 Jul 2022 16:35:17 -0400
Subject: [PATCH 5/6] fix --in-memory

---
 snapshotEngine/mainJob.yaml      |  2 +-
 snapshotEngine/zip-and-upload.sh | 13 +++++++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/snapshotEngine/mainJob.yaml b/snapshotEngine/mainJob.yaml
index d0a1e000e..f049a0105 100644
--- a/snapshotEngine/mainJob.yaml
+++ b/snapshotEngine/mainJob.yaml
@@ -197,7 +197,7 @@ spec:
 
                 tezos-node snapshot import \
                 /"${HISTORY_MODE}"-snapshot-cache-volume/"${ROLLING_SNAPSHOT_NAME}".rolling \
-                --in-memory
+                --in-memory \
                 --block "${BLOCK_HASH}" \
                 --config-file /home/tezos/.tezos-node/config.json \
                 --data-dir /rolling-tarball-restore/var/tezos/node/data
diff --git a/snapshotEngine/zip-and-upload.sh b/snapshotEngine/zip-and-upload.sh
index f51c4fd56..f84226316 100755
--- a/snapshotEngine/zip-and-upload.sh
+++ b/snapshotEngine/zip-and-upload.sh
@@ -417,16 +417,17 @@ fi
 touch base.json
 echo '[]' > "base.json"
 
-tmp=$(mktemp)
-cp base.json "${tmp}"
-
-aws s3 ls s3://"$NETWORK".xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | while read ITEM; do
-    jq --argjson file "$(curl https://$NETWORK.xtz-shots.io/$ITEM)" '. += [$file]' "${tmp}" > base.json && rm "${tmp}"
+aws s3 ls s3://"${NETWORK}".xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | grep -v base.json | while read ITEM; do
+    tmp=$(mktemp) && cp base.json "${tmp}" && jq --argjson file "$(curl  -S -s -o /dev/null https://"${NETWORK}".xtz-shots.io/$ITEM)" '. += [$file]' "${tmp}" > base.json
 done
 
+# aws s3 ls s3://$NETWORK.xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | grep -v base.json | while read ITEM; do
+#     echo $ITEM
+# done
+
 #Upload base.json
 if ! aws s3 cp base.json s3://"${S3_BUCKET}"/base.json; then
     printf "%s Upload base.json : Error uploading file base.json to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
 else
     printf "%s Upload base.json : File base.json sucessfully uploaded to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-fi
\ No newline at end of file
+fi

From 24d523b114597cc8a1520d512aaefd2844ad4556 Mon Sep 17 00:00:00 2001
From: orcutt989 <jimmy.orcutt@tqtezos.com>
Date: Tue, 12 Jul 2022 17:01:57 -0400
Subject: [PATCH 6/6] cleanup and add file loop

---
 snapshotEngine/zip-and-upload.sh | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/snapshotEngine/zip-and-upload.sh b/snapshotEngine/zip-and-upload.sh
index f84226316..722b80a95 100755
--- a/snapshotEngine/zip-and-upload.sh
+++ b/snapshotEngine/zip-and-upload.sh
@@ -413,14 +413,23 @@ fi
 
 # Build base.json from existing metadata files
 
+printf "##### BEGINNING OF BASE.JSON"
+
 # Create new base.json locally
 touch base.json
 echo '[]' > "base.json"
 
+printf "%s Building base.json... this may take a while." "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
 aws s3 ls s3://"${NETWORK}".xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | grep -v base.json | while read ITEM; do
-    tmp=$(mktemp) && cp base.json "${tmp}" && jq --argjson file "$(curl  -S -s -o /dev/null https://"${NETWORK}".xtz-shots.io/$ITEM)" '. += [$file]' "${tmp}" > base.json
+    tmp=$(mktemp) && cp base.json "${tmp}" && jq --argjson file "$(curl -s https://"${NETWORK}".xtz-shots.io/$ITEM)" '. += [$file]' "${tmp}" > base.json
 done
 
+# DEBUG
+
+printf "%s" "$(cat base.json || true)"
+printf "##### END OF BASE.JSON"
+# END DEBUG
+
 # aws s3 ls s3://$NETWORK.xtz-shots.io |  grep '\.json'| sort | awk '{print $4}' | awk -F '\\\\n' '{print $1}' | tr ' ' '\n' | grep -v base.json | while read ITEM; do
 #     echo $ITEM
 # done
@@ -430,4 +439,4 @@ if ! aws s3 cp base.json s3://"${S3_BUCKET}"/base.json; then
     printf "%s Upload base.json : Error uploading file base.json to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
 else
     printf "%s Upload base.json : File base.json sucessfully uploaded to S3.  \n" "$(date "+%Y-%m-%d %H:%M:%S" "$@")"
-fi
+fi
\ No newline at end of file