Handle Metadata from xtz-shots

charts/tezos/scripts/snapshot-importer.sh

@@ -1,4 +1,4 @@
-set -ex
+set -e
 
 bin_dir="/usr/local/bin"
 data_dir="/var/tezos"
@@ -7,6 +7,11 @@ node_data_dir="$node_dir/data"
 node="$bin_dir/octez-node"
 snapshot_file=${node_dir}/chain.snapshot
 
+if [ ! -f ${snapshot_file} ]; then
+ echo "No snapshot to import."
+ exit 0
+fi
+
 if [ -d ${node_data_dir}/context ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
@@ -16,8 +21,12 @@ fi
 
 cp -v /etc/tezos/config.json ${node_data_dir}
 
+if [ -f ${node_dir}/chain.snapshot.block_hash ]; then
+ block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)"
+fi
+
 ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \
- --network $CHAIN_NAME
+ --network $CHAIN_NAME ${block_hash_arg}
 find ${node_dir}
 
 rm -rvf ${snapshot_file}

charts/tezos/templates/_helpers.tpl

@@ -56,7 +56,7 @@
 {{- define "tezos.shouldDownloadSnapshot" -}}
  {{- if or (.Values.full_snapshot_url) (.Values.full_tarball_url)
  (.Values.rolling_snapshot_url) (.Values.rolling_tarball_url)
- (.Values.archive_tarball_url) }}
+ (.Values.archive_tarball_url) (.Values.snapshot_source) }}
  {{- if or (and (.Values.rolling_tarball_url) (.Values.rolling_snapshot_url))
  (and (.Values.full_tarball_url) (.Values.full_snapshot_url))
  }}

charts/tezos/templates/configs.yaml

@@ -16,6 +16,9 @@ data:
  ROLLING_SNAPSHOT_URL: "{{ .Values.rolling_snapshot_url }}"
  ROLLING_TARBALL_URL: "{{ .Values.rolling_tarball_url }}"
  ARCHIVE_TARBALL_URL: "{{ .Values.archive_tarball_url }}"
+ PREFER_TARBALLS: "{{ .Values.prefer_tarballs }}"
+ SNAPSHOT_SOURCE: "{{ .Values.snapshot_source }}"
+ OCTEZ_VERSION: "{{ .Values.images.octez }}"
  NODE_GLOBALS: |
 {{ .Values.node_globals | mustToPrettyJson | indent 4 }}
 

charts/tezos/values.yaml

@@ -273,17 +273,30 @@ signers: {}
 # ```
 # End Signers
 
-## Where full and rolling history mode nodes will get their Tezos snapshots from.
-full_snapshot_url: https://mainnet.xtz-shots.io/full
-rolling_snapshot_url: https://mainnet.xtz-shots.io/rolling
+# When spinning up nodes, tezos-k8s will attempt to download a snapshot from a
+# known source. This should be a url to a json metadata file in the format
+# xtz-shots uses. If you want to sync from scratch or for a private chain, set
+# to `null`.
+snapshot_source: https://xtz-shots.io/tezos-snapshots.json
 
-## Alternatively to a Tezos snapshot, you can download an LZ4-compressed
-## filesystem tar of a node's data directory by setting `archive_tarball_url`
-## and `rolling_tarball_url` fields to the URL of the file. NOTE:
-## `rolling_tarball_url` and `rolling_snapshot_url` are mutually exclusive and
-## cannot both be specified at the same time.
-archive_tarball_url: https://mainnet.xtz-shots.io/archive-tarball
-# rolling_tarball_url: "https://mainnet.xtz-shots.io/rolling-tarball
+# By default, tezos-k8s will download and unpack snapshots.
+# A tarball is a LZ4-compressed filesystem tar of a node's data directory.
+# You must trust the tarball provider to provide good data, as no check is
+# performed by the node.
+# If you prefer tarballs, set to "true" below.
+prefer_tarballs: false
+
+# By default, tezos-k8s will attempt to download the right artifact from
+# `snapshot_source` set above. You can override and hard-code a snapshot URL
+# source below. When any of the below variables are set, `snapshot_source` above
+# will be ignored for all artifact types.
+## NOTE: `*_tarball_url` and `*_snapshot_url` are mutually exclusive
+## and cannot both be specified at the same time.
+archive_tarball_url: null # e.g. https://mainnet.xtz-shots.io/archive-tarball
+full_snapshot_url: null
+full_tarball_url: null
+rolling_snapshot_url: null
+rolling_tarball_url: null
 
 # List of peers for nodes to connect to. Gets set under config.json `p2p` field
 bootstrap_peers: []

mkchain/tqchain/mkchain.py

@@ -178,10 +178,7 @@ def main():
  "zerotier_token": args.zerotier_token,
  },
  # Custom chains should not pull snapshots or tarballs
- "full_snapshot_url": None,
- "rolling_snapshot_url": None,
- "archive_tarball_url": None,
- "rolling_tarball_url": None,
+ "snapshot_source": None,
  "node_globals": {
  # Needs a quotedstring otherwise helm interprets "Y" as true and it does not work
  "env": {

test/charts/mainnet.expect.yaml

@@ -26,11 +26,14 @@ data:
  },
  "protocol_activation": null
  }
- FULL_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/full"
+ FULL_SNAPSHOT_URL: ""
  FULL_TARBALL_URL: ""
- ROLLING_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/rolling"
+ ROLLING_SNAPSHOT_URL: ""
  ROLLING_TARBALL_URL: ""
- ARCHIVE_TARBALL_URL: "https://mainnet.xtz-shots.io/archive-tarball"
+ ARCHIVE_TARBALL_URL: ""
+ PREFER_TARBALLS: "false"
+ SNAPSHOT_SOURCE: "https://xtz-shots.io/tezos-snapshots.json"
+ OCTEZ_VERSION: "tezos/tezos:v15-release"
  NODE_GLOBALS: |
  {
  "env": {}
@@ -317,7 +320,7 @@ spec:
  args:
  - "-c"
  - |
- set -ex
+ set -e
 
  bin_dir="/usr/local/bin"
  data_dir="/var/tezos"
@@ -326,6 +329,11 @@ spec:
  node="$bin_dir/octez-node"
  snapshot_file=${node_dir}/chain.snapshot
 
+ if [ ! -f ${snapshot_file} ]; then
+ echo "No snapshot to import."
+ exit 0
+ fi
+
  if [ -d ${node_data_dir}/context ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
@@ -335,8 +343,12 @@ spec:
 
  cp -v /etc/tezos/config.json ${node_data_dir}
 
+ if [ -f ${node_dir}/chain.snapshot.block_hash ]; then
+ block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)"
+ fi
+
  ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \
- --network $CHAIN_NAME
+ --network $CHAIN_NAME ${block_hash_arg}
  find ${node_dir}
 
  rm -rvf ${snapshot_file}

test/charts/mainnet2.expect.yaml

@@ -26,11 +26,14 @@ data:
  },
  "protocol_activation": null
  }
- FULL_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/full"
+ FULL_SNAPSHOT_URL: ""
  FULL_TARBALL_URL: ""
- ROLLING_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/rolling"
+ ROLLING_SNAPSHOT_URL: ""
  ROLLING_TARBALL_URL: ""
- ARCHIVE_TARBALL_URL: "https://mainnet.xtz-shots.io/archive-tarball"
+ ARCHIVE_TARBALL_URL: ""
+ PREFER_TARBALLS: "false"
+ SNAPSHOT_SOURCE: "https://xtz-shots.io/tezos-snapshots.json"
+ OCTEZ_VERSION: "tezos/tezos:v15-release"
  NODE_GLOBALS: |
  {
  "env": {
@@ -426,7 +429,7 @@ spec:
  args:
  - "-c"
  - |
- set -ex
+ set -e
 
  bin_dir="/usr/local/bin"
  data_dir="/var/tezos"
@@ -435,6 +438,11 @@ spec:
  node="$bin_dir/octez-node"
  snapshot_file=${node_dir}/chain.snapshot
 
+ if [ ! -f ${snapshot_file} ]; then
+ echo "No snapshot to import."
+ exit 0
+ fi
+
  if [ -d ${node_data_dir}/context ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
@@ -444,8 +452,12 @@ spec:
 
  cp -v /etc/tezos/config.json ${node_data_dir}
 
+ if [ -f ${node_dir}/chain.snapshot.block_hash ]; then
+ block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)"
+ fi
+
  ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \
- --network $CHAIN_NAME
+ --network $CHAIN_NAME ${block_hash_arg}
  find ${node_dir}
 
  rm -rvf ${snapshot_file}
@@ -789,7 +801,7 @@ spec:
  args:
  - "-c"
  - |
- set -ex
+ set -e
 
  bin_dir="/usr/local/bin"
  data_dir="/var/tezos"
@@ -798,6 +810,11 @@ spec:
  node="$bin_dir/octez-node"
  snapshot_file=${node_dir}/chain.snapshot
 
+ if [ ! -f ${snapshot_file} ]; then
+ echo "No snapshot to import."
+ exit 0
+ fi
+
  if [ -d ${node_data_dir}/context ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
@@ -807,8 +824,12 @@ spec:
 
  cp -v /etc/tezos/config.json ${node_data_dir}
 
+ if [ -f ${node_dir}/chain.snapshot.block_hash ]; then
+ block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)"
+ fi
+
  ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \
- --network $CHAIN_NAME
+ --network $CHAIN_NAME ${block_hash_arg}
  find ${node_dir}
 
  rm -rvf ${snapshot_file}

test/charts/private-chain.expect.yaml

@@ -103,6 +103,9 @@ data:
  ROLLING_SNAPSHOT_URL: ""
  ROLLING_TARBALL_URL: ""
  ARCHIVE_TARBALL_URL: ""
+ PREFER_TARBALLS: "false"
+ SNAPSHOT_SOURCE: ""
+ OCTEZ_VERSION: "tezos/tezos:v15-release"
  NODE_GLOBALS: |
  {
  "env": {}

test/charts/private-chain.in.yaml

@@ -72,9 +72,7 @@ activation:
  sc_rollup_max_available_messages: 1000000
 bootstrap_peers: []
 expected_proof_of_work: 0
-full_snapshot_url: null
-rolling_snapshot_url: null
-archive_tarball_url: null
+snapshot_source: null
 images:
  octez: 'tezos/tezos:v15-release'
 is_invitation: false

utils/Dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
+# TODO: update to 3.11 once the bug is fixed:
+# https://github.com/baking-bad/pytezos/issues/336
 ENV PYTHONUNBUFFERED=1
 
 #
@@ -14,7 +16,7 @@ RUN PIP="pip --no-cache install" \
  $APK_ADD --virtual .build-deps gcc python3-dev \
  libffi-dev musl-dev make \
  && $APK_ADD libsodium-dev libsecp256k1-dev gmp-dev \
- && $APK_ADD zeromq-dev \
+ && $APK_ADD zeromq-dev findmnt \
  && $PIP install base58 pynacl \
  && $PIP install mnemonic pytezos requests \
  && $PIP install pyblake2 pysodium flask \