diff --git a/charts/tezos/scripts/snapshot-importer.sh b/charts/tezos/scripts/snapshot-importer.sh index 4727601d4..6977fca4b 100644 --- a/charts/tezos/scripts/snapshot-importer.sh +++ b/charts/tezos/scripts/snapshot-importer.sh @@ -1,4 +1,4 @@ -set -ex +set -e bin_dir="/usr/local/bin" data_dir="/var/tezos" @@ -7,6 +7,11 @@ node_data_dir="$node_dir/data" node="$bin_dir/octez-node" snapshot_file=${node_dir}/chain.snapshot +if [ ! -f ${snapshot_file} ]; then + echo "No snapshot to import." + exit 0 +fi + if [ -d ${node_data_dir}/context ]; then echo "Blockchain has already been imported. If a tarball" echo "instead of a regular tezos snapshot was used, it was" @@ -16,8 +21,12 @@ fi cp -v /etc/tezos/config.json ${node_data_dir} +if [ -f ${node_dir}/chain.snapshot.block_hash ]; then + block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)" +fi + ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \ - --network $CHAIN_NAME + --network $CHAIN_NAME ${block_hash_arg} find ${node_dir} rm -rvf ${snapshot_file} diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl index dd99ebbd4..962a7e3e7 100644 --- a/charts/tezos/templates/_helpers.tpl +++ b/charts/tezos/templates/_helpers.tpl @@ -56,7 +56,7 @@ {{- define "tezos.shouldDownloadSnapshot" -}} {{- if or (.Values.full_snapshot_url) (.Values.full_tarball_url) (.Values.rolling_snapshot_url) (.Values.rolling_tarball_url) - (.Values.archive_tarball_url) }} + (.Values.archive_tarball_url) (.Values.snapshot_source) }} {{- if or (and (.Values.rolling_tarball_url) (.Values.rolling_snapshot_url)) (and (.Values.full_tarball_url) (.Values.full_snapshot_url)) }} diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml index ad0adf1d8..88b2872f7 100644 --- a/charts/tezos/templates/configs.yaml +++ b/charts/tezos/templates/configs.yaml @@ -16,6 +16,9 @@ data: ROLLING_SNAPSHOT_URL: "{{ .Values.rolling_snapshot_url }}" ROLLING_TARBALL_URL: "{{ .Values.rolling_tarball_url }}" ARCHIVE_TARBALL_URL: "{{ .Values.archive_tarball_url }}" + PREFER_TARBALLS: "{{ .Values.prefer_tarballs }}" + SNAPSHOT_SOURCE: "{{ .Values.snapshot_source }}" + OCTEZ_VERSION: "{{ .Values.images.octez }}" NODE_GLOBALS: | {{ .Values.node_globals | mustToPrettyJson | indent 4 }} diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index d2cfaa92a..4b155730b 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -273,17 +273,30 @@ signers: {} # ``` # End Signers -## Where full and rolling history mode nodes will get their Tezos snapshots from. -full_snapshot_url: https://mainnet.xtz-shots.io/full -rolling_snapshot_url: https://mainnet.xtz-shots.io/rolling +# When spinning up nodes, tezos-k8s will attempt to download a snapshot from a +# known source. This should be a url to a json metadata file in the format +# xtz-shots uses. If you want to sync from scratch or for a private chain, set +# to `null`. +snapshot_source: https://xtz-shots.io/tezos-snapshots.json -## Alternatively to a Tezos snapshot, you can download an LZ4-compressed -## filesystem tar of a node's data directory by setting `archive_tarball_url` -## and `rolling_tarball_url` fields to the URL of the file. NOTE: -## `rolling_tarball_url` and `rolling_snapshot_url` are mutually exclusive and -## cannot both be specified at the same time. -archive_tarball_url: https://mainnet.xtz-shots.io/archive-tarball -# rolling_tarball_url: "https://mainnet.xtz-shots.io/rolling-tarball +# By default, tezos-k8s will download and unpack snapshots. +# A tarball is a LZ4-compressed filesystem tar of a node's data directory. +# You must trust the tarball provider to provide good data, as no check is +# performed by the node. +# If you prefer tarballs, set to "true" below. +prefer_tarballs: false + +# By default, tezos-k8s will attempt to download the right artifact from +# `snapshot_source` set above. You can override and hard-code a snapshot URL +# source below. When any of the below variables are set, `snapshot_source` above +# will be ignored for all artifact types. +## NOTE: `*_tarball_url` and `*_snapshot_url` are mutually exclusive +## and cannot both be specified at the same time. +archive_tarball_url: null # e.g. https://mainnet.xtz-shots.io/archive-tarball +full_snapshot_url: null +full_tarball_url: null +rolling_snapshot_url: null +rolling_tarball_url: null # List of peers for nodes to connect to. Gets set under config.json `p2p` field bootstrap_peers: [] diff --git a/mkchain/tqchain/mkchain.py b/mkchain/tqchain/mkchain.py index 09dd96e48..704b03adb 100644 --- a/mkchain/tqchain/mkchain.py +++ b/mkchain/tqchain/mkchain.py @@ -178,10 +178,7 @@ def main(): "zerotier_token": args.zerotier_token, }, # Custom chains should not pull snapshots or tarballs - "full_snapshot_url": None, - "rolling_snapshot_url": None, - "archive_tarball_url": None, - "rolling_tarball_url": None, + "snapshot_source": None, "node_globals": { # Needs a quotedstring otherwise helm interprets "Y" as true and it does not work "env": { diff --git a/test/charts/mainnet.expect.yaml b/test/charts/mainnet.expect.yaml index d36c25768..4c6b923f6 100644 --- a/test/charts/mainnet.expect.yaml +++ b/test/charts/mainnet.expect.yaml @@ -26,11 +26,14 @@ data: }, "protocol_activation": null } - FULL_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/full" + FULL_SNAPSHOT_URL: "" FULL_TARBALL_URL: "" - ROLLING_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/rolling" + ROLLING_SNAPSHOT_URL: "" ROLLING_TARBALL_URL: "" - ARCHIVE_TARBALL_URL: "https://mainnet.xtz-shots.io/archive-tarball" + ARCHIVE_TARBALL_URL: "" + PREFER_TARBALLS: "false" + SNAPSHOT_SOURCE: "https://xtz-shots.io/tezos-snapshots.json" + OCTEZ_VERSION: "tezos/tezos:v15-release" NODE_GLOBALS: | { "env": {} @@ -317,7 +320,7 @@ spec: args: - "-c" - | - set -ex + set -e bin_dir="/usr/local/bin" data_dir="/var/tezos" @@ -326,6 +329,11 @@ spec: node="$bin_dir/octez-node" snapshot_file=${node_dir}/chain.snapshot + if [ ! -f ${snapshot_file} ]; then + echo "No snapshot to import." + exit 0 + fi + if [ -d ${node_data_dir}/context ]; then echo "Blockchain has already been imported. If a tarball" echo "instead of a regular tezos snapshot was used, it was" @@ -335,8 +343,12 @@ spec: cp -v /etc/tezos/config.json ${node_data_dir} + if [ -f ${node_dir}/chain.snapshot.block_hash ]; then + block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)" + fi + ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \ - --network $CHAIN_NAME + --network $CHAIN_NAME ${block_hash_arg} find ${node_dir} rm -rvf ${snapshot_file} diff --git a/test/charts/mainnet2.expect.yaml b/test/charts/mainnet2.expect.yaml index ad1069047..07b71ee3e 100644 --- a/test/charts/mainnet2.expect.yaml +++ b/test/charts/mainnet2.expect.yaml @@ -26,11 +26,14 @@ data: }, "protocol_activation": null } - FULL_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/full" + FULL_SNAPSHOT_URL: "" FULL_TARBALL_URL: "" - ROLLING_SNAPSHOT_URL: "https://mainnet.xtz-shots.io/rolling" + ROLLING_SNAPSHOT_URL: "" ROLLING_TARBALL_URL: "" - ARCHIVE_TARBALL_URL: "https://mainnet.xtz-shots.io/archive-tarball" + ARCHIVE_TARBALL_URL: "" + PREFER_TARBALLS: "false" + SNAPSHOT_SOURCE: "https://xtz-shots.io/tezos-snapshots.json" + OCTEZ_VERSION: "tezos/tezos:v15-release" NODE_GLOBALS: | { "env": { @@ -426,7 +429,7 @@ spec: args: - "-c" - | - set -ex + set -e bin_dir="/usr/local/bin" data_dir="/var/tezos" @@ -435,6 +438,11 @@ spec: node="$bin_dir/octez-node" snapshot_file=${node_dir}/chain.snapshot + if [ ! -f ${snapshot_file} ]; then + echo "No snapshot to import." + exit 0 + fi + if [ -d ${node_data_dir}/context ]; then echo "Blockchain has already been imported. If a tarball" echo "instead of a regular tezos snapshot was used, it was" @@ -444,8 +452,12 @@ spec: cp -v /etc/tezos/config.json ${node_data_dir} + if [ -f ${node_dir}/chain.snapshot.block_hash ]; then + block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)" + fi + ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \ - --network $CHAIN_NAME + --network $CHAIN_NAME ${block_hash_arg} find ${node_dir} rm -rvf ${snapshot_file} @@ -789,7 +801,7 @@ spec: args: - "-c" - | - set -ex + set -e bin_dir="/usr/local/bin" data_dir="/var/tezos" @@ -798,6 +810,11 @@ spec: node="$bin_dir/octez-node" snapshot_file=${node_dir}/chain.snapshot + if [ ! -f ${snapshot_file} ]; then + echo "No snapshot to import." + exit 0 + fi + if [ -d ${node_data_dir}/context ]; then echo "Blockchain has already been imported. If a tarball" echo "instead of a regular tezos snapshot was used, it was" @@ -807,8 +824,12 @@ spec: cp -v /etc/tezos/config.json ${node_data_dir} + if [ -f ${node_dir}/chain.snapshot.block_hash ]; then + block_hash_arg="--block $(cat ${node_dir}/chain.snapshot.block_hash)" + fi + ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} \ - --network $CHAIN_NAME + --network $CHAIN_NAME ${block_hash_arg} find ${node_dir} rm -rvf ${snapshot_file} diff --git a/test/charts/private-chain.expect.yaml b/test/charts/private-chain.expect.yaml index 3479868c0..51be255cc 100644 --- a/test/charts/private-chain.expect.yaml +++ b/test/charts/private-chain.expect.yaml @@ -103,6 +103,9 @@ data: ROLLING_SNAPSHOT_URL: "" ROLLING_TARBALL_URL: "" ARCHIVE_TARBALL_URL: "" + PREFER_TARBALLS: "false" + SNAPSHOT_SOURCE: "" + OCTEZ_VERSION: "tezos/tezos:v15-release" NODE_GLOBALS: | { "env": {} diff --git a/test/charts/private-chain.in.yaml b/test/charts/private-chain.in.yaml index 39cf6b577..db1f4812c 100644 --- a/test/charts/private-chain.in.yaml +++ b/test/charts/private-chain.in.yaml @@ -72,9 +72,7 @@ activation: sc_rollup_max_available_messages: 1000000 bootstrap_peers: [] expected_proof_of_work: 0 -full_snapshot_url: null -rolling_snapshot_url: null -archive_tarball_url: null +snapshot_source: null images: octez: 'tezos/tezos:v15-release' is_invitation: false diff --git a/utils/Dockerfile b/utils/Dockerfile index 7817a1d24..84e5c125d 100644 --- a/utils/Dockerfile +++ b/utils/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine +# TODO: update to 3.11 once the bug is fixed: +# https://github.com/baking-bad/pytezos/issues/336 ENV PYTHONUNBUFFERED=1 # @@ -14,7 +16,7 @@ RUN PIP="pip --no-cache install" \ $APK_ADD --virtual .build-deps gcc python3-dev \ libffi-dev musl-dev make \ && $APK_ADD libsodium-dev libsecp256k1-dev gmp-dev \ - && $APK_ADD zeromq-dev \ + && $APK_ADD zeromq-dev findmnt \ && $PIP install base58 pynacl \ && $PIP install mnemonic pytezos requests \ && $PIP install pyblake2 pysodium flask \ diff --git a/utils/config-generator.py b/utils/config-generator.py index 9df93a22f..7b0dcf62b 100755 --- a/utils/config-generator.py +++ b/utils/config-generator.py @@ -2,8 +2,10 @@ import collections import json import os +import re import requests import socket +import sys from grp import getgrnam from hashlib import blake2b from pathlib import Path @@ -14,7 +16,7 @@ from pytezos import pytezos from base58 import b58encode_check -with open('/etc/secret-volume/ACCOUNTS', 'r') as secret_file: +with open("/etc/secret-volume/ACCOUNTS", "r") as secret_file: ACCOUNTS = json.loads(secret_file.read()) CHAIN_PARAMS = json.loads(os.environ["CHAIN_PARAMS"]) DATA_DIR = "/var/tezos/node/data" @@ -141,17 +143,30 @@ def main(): "ERROR: No bootstrap peers found for this non-bootstrap node" ) - config_json = json.dumps( - create_node_config_json( - bootstrap_peers, - my_zerotier_ip, - ), + node_config = create_node_config_json( + bootstrap_peers, + my_zerotier_ip, + ) + node_config_json = json.dumps( + node_config, + indent=2, + ) + node_snapshot_config = create_node_snapshot_config_json( + node_config["shell"]["history_mode"] + ) + node_snapshot_config_json = json.dumps( + node_snapshot_config, indent=2, ) print("Generated config.json :") - print(config_json) + print(node_config_json) with open("/etc/tezos/config.json", "w") as json_file: - print(config_json, file=json_file) + print(node_config_json, file=json_file) + if node_snapshot_config: + print("Generated snapshot_config.json :") + print(node_snapshot_config_json) + with open("/var/tezos/snapshot_config.json", "w") as json_file: + print(node_snapshot_config_json, file=json_file) # If NETWORK_CONFIG["genesis"]["block"] hasn't been specified, we generate a @@ -249,7 +264,9 @@ def verify_this_bakers_account(accounts): # We can count on accounts[acct]["type"] because import_keys will # fill it in when it is missing. if not (accounts[acct]["type"] == "secret" or signer): - raise Exception(f"ERROR: Either a secret key or a signer_url should be provided for {acct}") + raise Exception( + f"ERROR: Either a secret key or a signer_url should be provided for {acct}" + ) # @@ -282,7 +299,9 @@ def fill_in_missing_keys(all_accounts): for account_name, account_values in all_accounts.items(): if "type" in account_values: - raise Exception("Deprecated field 'type' passed by helm, but helm should have pruned it.") + raise Exception( + "Deprecated field 'type' passed by helm, but helm should have pruned it." + ) account_key = account_values.get("key") if account_key == None: @@ -324,7 +343,9 @@ def expose_secret_key(account_name): def pod_requires_secret_key(account_values): - return MY_POD_TYPE in ["activating", "signing"] and "signer_url" not in account_values + return ( + MY_POD_TYPE in ["activating", "signing"] and "signer_url" not in account_values + ) # @@ -521,9 +542,11 @@ def get_genesis_pubkey(): genesis_pubkey = pubkey["value"]["key"] break if not genesis_pubkey: - raise Exception("ERROR: Couldn't find the genesis_pubkey. " + - "This generally happens if you forgot to " + - "define an account for the activation account") + raise Exception( + "ERROR: Couldn't find the genesis_pubkey. " + + "This generally happens if you forgot to " + + "define an account for the activation account" + ) return genesis_pubkey @@ -552,7 +575,7 @@ def create_node_config_json( "data-dir": DATA_DIR, "rpc": { "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], - "acl": [ { "address": os.getenv('MY_POD_IP'), "blacklist": [] } ] + "acl": [{"address": os.getenv("MY_POD_IP"), "blacklist": []}], }, "p2p": { "bootstrap-peers": bootstrap_peers, @@ -600,5 +623,94 @@ def create_node_config_json( return node_config +def create_node_snapshot_config_json(history_mode): + """Create this node's snapshot config""" + + network_name = NETWORK_CONFIG.get("chain_name") + prefer_tarballs = os.environ.get("PREFER_TARBALLS", "").lower() in ("true", "1", "t") + artifact_type = "tarball" if prefer_tarballs else "tezos-snapshot" + rolling_tarball_url = os.environ.get("ROLLING_TARBALL_URL") + full_tarball_url = os.environ.get("FULL_TARBALL_URL") + archive_tarball_url = os.environ.get("ARCHIVE_TARBALL_URL") + rolling_snapshot_url = os.environ.get("ROLLING_SNAPSHOT_URL") + full_snapshot_url = os.environ.get("FULL_SNAPSHOT_URL") + if ( + rolling_tarball_url + or full_tarball_url + or rolling_snapshot_url + or full_snapshot_url + or archive_tarball_url + ): + print("Snapshot or tarball URL found, will ignore snapshot_source") + match history_mode: + case "rolling": + if rolling_tarball_url: + return {"url": rolling_tarball_url, "artifact_type": "tarball"} + elif rolling_snapshot_url: + return { + "url": rolling_snapshot_url, + "artifact_type": "tezos-snapshot", + } + return + case "full": + if full_tarball_url: + return {"url": full_tarball_url, "artifact_type": "tarball"} + elif full_snapshot_url: + return {"url": full_snapshot_url, "artifact_type": "tezos-snapshot"} + return + case "archive": + if archive_tarball_url: + return {"url": archive_tarball_url, "artifact_type": "tarball"} + return + case _: + print(f"Error: history mode {history_mode} is not known.") + sys.exit(1) + + if "images" in MY_POD_CLASS and "octez" in MY_POD_CLASS["images"]: + octez_container_version = MY_POD_CLASS["images"]["octez"] + else: + octez_container_version = os.environ.get("OCTEZ_VERSION") + snapshot_source = os.environ.get("SNAPSHOT_SOURCE") + if snapshot_source: + try: + all_snapshots = requests.get(snapshot_source).json() + except Exception as e: + print(f"Error while fetching {snapshot_source}: {e}") + return + else: + return + try: + octez_long_version = octez_container_version.split(":")[1] + octez_version_re = re.search(r"v(\d+)", octez_long_version) + octez_version = octez_version_re and octez_version_re.group(1) + except Exception: + octez_version = None + + print( + f""" +Searching for snapshots from {snapshot_source} +with history mode {history_mode} +and artifact type {artifact_type} +and chain name {network_name} +and octez version {octez_version}. + """ + ) + # find snapshot matching all the requested fields + matching_snapshots = [ + s + for s in all_snapshots + if s.get("history_mode") == history_mode + and s.get("artifact_type") == artifact_type + and s.get("chain_name") == network_name + ] + if octez_version: + matching_snapshots = [ + s for s in matching_snapshots if octez_version in s.get("tezos_version", "") + ] + matching_snapshots = sorted(matching_snapshots, key=lambda s: s.get("block_height")) + + return matching_snapshots[-1] if len(matching_snapshots) else None + + if __name__ == "__main__": main() diff --git a/utils/snapshot-downloader.sh b/utils/snapshot-downloader.sh index 31452311b..9a2a10bb6 100755 --- a/utils/snapshot-downloader.sh +++ b/utils/snapshot-downloader.sh @@ -19,48 +19,74 @@ fi echo "Did not find a pre-existing blockchain." -my_nodes_history_mode=$(< /etc/tezos/config.json jq -r " - .shell.history_mode - |if type == \"object\" then - (keys|.[0]) - else . - end") - -echo "My nodes history mode: '$my_nodes_history_mode'" - -snapshot_url="" -tarball_url="" -case "$my_nodes_history_mode" in - full) snapshot_url="$FULL_SNAPSHOT_URL" - tarball_url="$FULL_TARBALL_URL";; - - rolling) snapshot_url="$ROLLING_SNAPSHOT_URL" - tarball_url="$ROLLING_TARBALL_URL";; - - archive) tarball_url="$ARCHIVE_TARBALL_URL";; - - *) echo "Invalid node history mode: '$my_nodes_history_mode'" - exit 1;; -esac - -if [ -z "$snapshot_url" ] && [ -z "$tarball_url" ]; then - echo "ERROR: No snapshot or tarball url specified." - exit 1 +if [ ! -f ${data_dir}/snapshot_config.json ]; then + echo "No snapshot config found, nothing to do." + exit 0 fi -if [ -n "$snapshot_url" ] && [ -n "$tarball_url" ]; then - echo "ERROR: Either only a snapshot or tarball url may be specified per Tezos node history mode." -fi +echo "Tezos snapshot config is:" +cat ${data_dir}/snapshot_config.json +artifact_url=$(cat ${data_dir}/snapshot_config.json | jq -r '.url') +artifact_type=$(cat ${data_dir}/snapshot_config.json | jq -r '.artifact_type') mkdir -p "$node_data_dir" -if [ -n "$snapshot_url" ]; then - echo "Downloading $snapshot_url" +download() { + # Smart Downloading function. When relevant metadata is accessible, it: + # * checks that there is enough space to download the file + # * verifies the sha256sum + filesize_bytes=$(cat ${data_dir}/snapshot_config.json | jq -r '.filesize_bytes // empty') + sha256=$(cat ${data_dir}/snapshot_config.json | jq -r '.sha256 // empty') + if [ ! -z "${filesize_bytes}" ]; then + free_space=$(findmnt -bno size -T ${data_dir}) + echo "Free space available in filesystem: ${free_space}" >&2 + if [ "${filesize_bytes}" -gt "${free_space}" ]; then + echo "Error: not enough disk space available (${free_space} bytes) to download artifact of size ${filesize_bytes} bytes." >&2 + touch ${data_dir}/disk_space_failed + return 1 + else + echo "There is sufficient free space to download the artifact of size ${filesize_bytes}." >&2 + fi + fi + curl -LfsS $1 | tee >(sha256sum > ${snapshot_file}.sha256sum) + if [ ! -z "${sha256}" ]; then + if [ "${sha256}" != "$(cat ${snapshot_file}.sha256sum | head -c 64)" ]; then + echo "Error: sha256 checksum of the downloaded file did not match checksum from metadata file." >&2 + touch ${data_dir}/sha256sum_failed + return 1 + else + echo "Snapshot sha256sum check successful." >&2 + fi + fi +} + +if [ "${artifact_type}" == "tezos-snapshot" ]; then + echo "Downloading $artifact_url" echo '{ "version": "0.0.4" }' > "$node_dir/version.json" - curl -LfsS -o "$snapshot_file" "$snapshot_url" -elif [ -n "$tarball_url" ]; then - echo "Downloading and extracting tarball from $tarball_url" - curl -LfsS "$tarball_url" | lz4 -d | tar -x -C "$data_dir" + block_hash=$(cat ${data_dir}/snapshot_config.json | jq -r '.block_hash // empty') + download "$artifact_url" > "$snapshot_file" + if [ -f "${data_dir}/sha256sum_failed" ]; then + # sha256 failure + rm -rvf ${snapshot_file} + rm -rvf "${data_dir}/sha256sum_failed" + exit 1 + fi + if [ ! -z "${block_hash}" ]; then + echo ${block_hash} > ${snapshot_file}.block_hash + fi +elif [ "${artifact_type}" == "tarball" ]; then + echo "Downloading and extracting tarball from $artifact_url" + download "$artifact_url" | lz4 -d | tar -x -C "$data_dir" + if [ -f "${data_dir}/sha256sum_failed" ]; then + echo "sha256 check failed, deleting data" + rm -rvf "${node_data_dir}" + rm -rvf "${data_dir}/sha256sum_failed" + exit 1 + fi +fi +if [ -f "${data_dir}/disk_space_failed" ]; then + rm -rvf "${data_dir}/disk_space_failed" + exit 1 fi chown -R 1000 "$data_dir"