Persist identity and peers when using local storage (#547)

* Persist identity and peers when using local storage

Local storage is great because it's faster, and we can download tarballs
in minutes at every boot and have tolerable downtimes.

But at each boot, the node regenerates its identity. We already have a
mechanism to pre-populate identities in the node yaml, but it's
cumbersome.

Instead, with this PR, we create a mini-pvc of 1Gi to store node
identities, whenever local storage is in use.

This way, local storage nodes always boot quickly, except the first time
they boot, and there is no configuration burden. It is also possible to
scale up the number of nodes without having to prepare identities.

I tested private chain and ghostnet node, in both local and non-local
storage. It behaves as intended. I also tried to pre-populate an
identity in the values file in local storage and confirmed that it is
not broken.

* completely remove chmod 777

* fix error in snapshot-importer: check file existence

* fix snapshot importer as well

* fix tests

* Update charts/tezos/scripts/upgrade-storage.sh

Co-authored-by: Aryeh Harris <harryttd@users.noreply.github.com>

* fix tests, add comment for persistent dir per review

* typo

* fix perms issue in the absence of 777

---------

Co-authored-by: Aryeh Harris <harryttd@users.noreply.github.com>

charts/tezos/scripts/snapshot-importer.sh

@@ -12,7 +12,7 @@ if [ ! -f ${snapshot_file} ]; then
  exit 0
 fi
 
-if [ -d ${node_data_dir}/context ]; then
+if [ -e ${node_data_dir}/context/store.dict ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
  echo "imported in the snapshot-downloader container."

charts/tezos/scripts/upgrade-storage.sh

@@ -1,8 +1,8 @@
 set -ex
 
-if [ ! -e /var/tezos/node/data/context ]
+if [ ! -e /var/tezos/node/data/context/store.dict ]
 then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
 fi
 octez-node upgrade storage --config /etc/tezos/config.json

charts/tezos/templates/_containers.tpl

@@ -150,6 +150,10 @@
  name: config-volume
  - mountPath: /var/tezos
  name: var-volume
+ {{- if .local_storage }}
+ - mountPath: /var/persistent
+ name: persistent-volume
+ {{- end }}
  {{- if .with_secret }}
  - mountPath: /etc/secret-volume
  name: tezos-accounts
@@ -264,6 +268,7 @@
  "type" "octez-node"
  "image" "octez"
  "with_config" 0
+ "local_storage" $.node_vals.local_storage
  "resources" $.node_vals.resources
  ) | nindent 0 }}
 {{- end }}

charts/tezos/templates/nodes.yaml

@@ -66,10 +66,16 @@ spec:
 {{- if $.node_vals.local_storage | default false }}
  - emptyDir: {}
  name: var-volume
-{{- else }}
+{{- end }}
  volumeClaimTemplates:
  - metadata:
+{{- if $.node_vals.local_storage | default false }}
+ # since we are using local storage, deploy a small persistent volume
+ # to store identity.json and peers.json
+ name: persistent-volume
+{{- else }}
  name: var-volume
+{{- end }}
  namespace: {{ $.Release.Namespace }}
  spec:
  accessModes:
@@ -79,6 +85,9 @@ spec:
  {{- end }}
  resources:
  requests:
+{{- if $.node_vals.local_storage | default false }}
+ storage: "1Gi"
+{{- else }}
  storage: {{ default "15Gi" $.node_vals.storage_size }}
 {{- end }}
 ---

test/charts/mainnet.expect.yaml

@@ -341,7 +341,7 @@ spec:
  exit 0
  fi
  
- if [ -d ${node_data_dir}/context ]; then
+ if [ -e ${node_data_dir}/context/store.dict ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
  echo "imported in the snapshot-downloader container."
@@ -392,9 +392,9 @@ spec:
  - |
  set -ex
  
- if [ ! -e /var/tezos/node/data/context ]
+ if [ ! -e /var/tezos/node/data/context/store.dict ]
  then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
  fi
  octez-node upgrade storage --config /etc/tezos/config.json

test/charts/mainnet2.expect.yaml

@@ -450,7 +450,7 @@ spec:
  exit 0
  fi
  
- if [ -d ${node_data_dir}/context ]; then
+ if [ -e ${node_data_dir}/context/store.dict ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
  echo "imported in the snapshot-downloader container."
@@ -503,9 +503,9 @@ spec:
  - |
  set -ex
  
- if [ ! -e /var/tezos/node/data/context ]
+ if [ ! -e /var/tezos/node/data/context/store.dict ]
  then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
  fi
  octez-node upgrade storage --config /etc/tezos/config.json
@@ -828,7 +828,7 @@ spec:
  exit 0
  fi
  
- if [ -d ${node_data_dir}/context ]; then
+ if [ -e ${node_data_dir}/context/store.dict ]; then
  echo "Blockchain has already been imported. If a tarball"
  echo "instead of a regular tezos snapshot was used, it was"
  echo "imported in the snapshot-downloader container."
@@ -881,9 +881,9 @@ spec:
  - |
  set -ex
  
- if [ ! -e /var/tezos/node/data/context ]
+ if [ ! -e /var/tezos/node/data/context/store.dict ]
  then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
  fi
  octez-node upgrade storage --config /etc/tezos/config.json

test/charts/private-chain.expect.yaml

@@ -440,9 +440,9 @@ spec:
  - |
  set -ex
  
- if [ ! -e /var/tezos/node/data/context ]
+ if [ ! -e /var/tezos/node/data/context/store.dict ]
  then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
  fi
  octez-node upgrade storage --config /etc/tezos/config.json
@@ -1013,9 +1013,9 @@ spec:
  - |
  set -ex
  
- if [ ! -e /var/tezos/node/data/context ]
+ if [ ! -e /var/tezos/node/data/context/store.dict ]
  then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
  fi
  octez-node upgrade storage --config /etc/tezos/config.json
@@ -1217,9 +1217,9 @@ spec:
  - |
  set -ex
  
- if [ ! -e /var/tezos/node/data/context ]
+ if [ ! -e /var/tezos/node/data/context/store.dict ]
  then
- printf "No context in data dir found, probably initial start, doing nothing."
+ printf "No store in data dir found, probably initial start, doing nothing."
  exit 0
  fi
  octez-node upgrade storage --config /etc/tezos/config.json

utils/config-generator.sh

@@ -6,7 +6,7 @@ cat /etc/tezos/data/config.json
 echo ------------------------------------------------------------
 
 mkdir -p /var/tezos/client
-chmod -R 777 /var/tezos
+mkdir -p /var/tezos/node/data
 set -e
 python3 /config-generator.py "$@"
 set +e
@@ -17,6 +17,15 @@ set +e
 # tezos docker image.
 
 MY_CLASS=$(echo $NODES | jq -r ".\"${MY_NODE_CLASS}\"")
+
+# Set up symlinks for local storage
+local_storage=$(echo $MY_CLASS | jq -r ".local_storage")
+if [ "${local_storage}" == "true" ]; then
+ echo '{ "version": "2.0" }' > /var/tezos/node/data/version.json
+ ln -s /var/persistent/peers.json /var/tezos/node/data/peers.json
+ ln -s /var/persistent/identity.json /var/tezos/node/data/identity.json
+fi
+
 AM_I_BAKER=0
 if [ "$MY_CLASS" != null ]; then
  AM_I_BAKER=$(echo $MY_CLASS | \
@@ -39,3 +48,6 @@ if [ "$AM_I_BAKER" -eq 1 ]; then
 
  echo "$my_baker_account" > /etc/tezos/baker-account
 fi
+
+# make sure tezos user owns everything in /var/tezos
+chown -R 1000:1000 /var/tezos

utils/snapshot-downloader.sh

@@ -12,7 +12,7 @@ if [ ! -d "$data_dir" ]; then
  exit 1
 fi
 
-if [ -d "$node_data_dir/context" ]; then
+if [ -e "$node_data_dir/context/store.dict" ]; then
  echo "Blockchain has already been imported. Exiting."
  exit 0
 fi