diff --git a/charts/pyrometer/scripts/pyrometer_exporter.py b/charts/pyrometer/scripts/pyrometer_exporter.py index 4c2b61fbe..cdda7b21e 100644 --- a/charts/pyrometer/scripts/pyrometer_exporter.py +++ b/charts/pyrometer/scripts/pyrometer_exporter.py @@ -4,18 +4,20 @@ import datetime import logging -log = logging.getLogger('werkzeug') + +log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) application = Flask(__name__) unhealthy_bakers = set() -@application.route('/pyrometer_webhook', methods=['POST']) + +@application.route("/pyrometer_webhook", methods=["POST"]) def pyrometer_webhook(): - ''' + """ Receive all events from pyrometer - ''' + """ for msg in request.get_json(): if msg["kind"] == "baker_unhealthy": print(f"Baker {msg['baker']} is unhealthy") @@ -26,14 +28,16 @@ def pyrometer_webhook(): return "Webhook received" -@application.route('/metrics', methods=['GET']) + +@application.route("/metrics", methods=["GET"]) def prometheus_metrics(): - ''' + """ Prometheus endpoint - ''' - return f'''# total number of monitored bakers that are currently unhealthy + """ + return f"""# total number of monitored bakers that are currently unhealthy pyrometer_unhealthy_bakers_total {len(unhealthy_bakers)} -''' +""" + if __name__ == "__main__": - application.run(host = "0.0.0.0", port = 31732, debug = False) + application.run(host="0.0.0.0", port=31732, debug=False) diff --git a/charts/tezos/Chart.lock b/charts/tezos/Chart.lock deleted file mode 100644 index 6a1bd656f..000000000 --- a/charts/tezos/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: authenticated-rpc-tezos-chain - repository: file://../rpc-auth - version: 0.0.0 -digest: sha256:359e854ddadc434084a5894da498d9170a778a6aa345414f97d4c570eaf9b99b -generated: "2021-01-11T16:20:15.112419-05:00" diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl index 962a7e3e7..0f07fd6c6 100644 --- a/charts/tezos/templates/_helpers.tpl +++ b/charts/tezos/templates/_helpers.tpl @@ -112,11 +112,21 @@ metadata: {{- end }} {{- end }} +{{/* + Has per-block votes defined? +*/}} +{{- define "tezos.hasPerBlockVotes" }} + {{- range .Values.protocols }} + {{- if .vote }} + {{- "true" }} + {{- end }} + {{- end }} +{{- end }} + {{/* Should deploy TZKT indexer? */}} {{- define "tezos.shouldDeployTzktIndexer" -}} - {{- $indexers := .Values.indexers | default dict }} {{- if $indexers.tzkt }} {{- $tzkt_config := $indexers.tzkt.config | default dict }} @@ -126,16 +136,17 @@ metadata: {{- "" }} {{- end }} {{- end }} - {{- end }} {{/* - Has per-block votes defined? + Checks if `bcdIndexer` has `rpcUrl` and `dbPassword` set. + Returns the true type or empty string which is falsey. */}} -{{- define "tezos.hasPerBlockVotes" }} - {{- range .Values.protocols }} - {{- if .vote }} - {{- "true" }} - {{- end }} +{{- define "tezos.shouldDeployBcdIndexer" -}} + {{- $indexConfig := .Values.bcdIndexer | default dict }} + {{- if and $indexConfig.indexerRpcUrl $indexConfig.dbPassword }} + {{- "true" }} + {{- else }} + {{- "" }} {{- end }} {{- end }} diff --git a/charts/tezos/templates/bcd_indexer.yaml b/charts/tezos/templates/bcd_indexer.yaml new file mode 100644 index 000000000..c3aff9392 --- /dev/null +++ b/charts/tezos/templates/bcd_indexer.yaml @@ -0,0 +1,96 @@ +{{- if (include "tezos.shouldDeployBcdIndexer" .) }} + +apiVersion: v1 +kind: Service +metadata: + name: bcd + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 14000 + name: bcd + targetPort: bcd + selector: + app: {{ .Values.bcd_indexer_statefulset.name }} + type: NodePort + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: bcd-env + namespace: {{ .Release.Namespace }} +data: + BCD_ENV: sandbox + GIN_MODE: debug + DB_HOSTNAME: localhost + SANDBOX_NODE_URI: {{ .Values.bcdIndexer.indexerRpcUrl }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: db-creds + namespace: {{ .Release.Namespace }} +data: + POSTGRES_USER: {{ b64enc .Values.bcdIndexer.dbUser }} + POSTGRES_PASSWORD: {{ b64enc .Values.bcdIndexer.dbPassword }} + POSTGRES_DB: {{ b64enc .Values.bcdIndexer.dbName }} + PGDATA: {{ b64enc "/var/lib/postgresql/data/pgdata" }} + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Values.bcd_indexer_statefulset.name }} + namespace: {{ .Release.Namespace }} +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + app: {{ .Values.bcd_indexer_statefulset.name }} + serviceName: "" + template: + metadata: + labels: + app: {{ .Values.bcd_indexer_statefulset.name }} + spec: + containers: + - image: {{ .Values.bcdIndexer.images.api }} + name: api + ports: + - containerPort: 14000 + name: bcd + envFrom: + - configMapRef: + name: bcd-env + - secretRef: + name: db-creds + - image: {{ .Values.bcdIndexer.images.indexer }} + name: indexer + envFrom: + - configMapRef: + name: bcd-env + - secretRef: + name: db-creds + - image: postgres:14 + name: db + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: indexer-db + envFrom: + - secretRef: + name: db-creds + volumeClaimTemplates: + - metadata: + name: indexer-db + spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.bcdIndexer.storageClassName }} + resources: + requests: + storage: "300Gi" + +{{- end }} diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 4b155730b..df3ef5a92 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -18,6 +18,8 @@ tezos_k8s_images: ## scripts that will look up some of these values. They should not be modified. tzkt_indexer_statefulset: name: tzkt-indexer +bcd_indexer_statefulset: + name: bcd-indexer signer_statefulset: name: tezos-signer pod_type: signing @@ -264,7 +266,7 @@ serviceMonitor: # # By default no signer is configured: signers: {} -# Here is an example of octez signer config. When set, the +# Here is an example of octez signer config. When set, the # ``` # signers: # tezos-signer-0: @@ -449,8 +451,8 @@ protocols: ## Deploy an indexer with the chain. An indexer puts the chain ## contents in a database for efficient indexing. Most dapps need it. ## Supported indexers: -## * tzkt https://github.com/baking-bad/tzkt -# +## * tzkt: https://github.com/baking-bad/tzkt +## * bcd: https://github.com/baking-bad/bcdhub ## Templatized indexer configuration values go in the config field. ## # indexers: @@ -489,3 +491,14 @@ protocols: # db_password: bar # # api_log_level: Debug # # indexer_log_level: Debug + +bcdIndexer: + images: + api: ghcr.io/baking-bad/bcdhub-api:4.4.0 + indexer: ghcr.io/baking-bad/bcdhub-indexer:4.4.0 + # dbName: bcd + # dbPassword: bcd-password + # dbUser: bcd + # storageClassName: "" + # indexerRpcUrl: http://archive-baking-node-0.archive-baking-node:8732 + diff --git a/mkchain/tqchain/mkchain.py b/mkchain/tqchain/mkchain.py index 704b03adb..fd0b5398a 100644 --- a/mkchain/tqchain/mkchain.py +++ b/mkchain/tqchain/mkchain.py @@ -82,6 +82,7 @@ def quoted_scalar(dumper, data): # a representer to force quotations on scalars }, } + # python versions < 3.8 doesn't have "extend" action class ExtendAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): diff --git a/snapshotEngine/getAllSnapshotMetadata.py b/snapshotEngine/getAllSnapshotMetadata.py index 2527d3018..70ec6db8f 100644 --- a/snapshotEngine/getAllSnapshotMetadata.py +++ b/snapshotEngine/getAllSnapshotMetadata.py @@ -3,10 +3,10 @@ import urllib, json import urllib.request -allSubDomains = os.environ['ALL_SUBDOMAINS'].split(",") -snapshotWebsiteBaseDomain = os.environ['SNAPSHOT_WEBSITE_DOMAIN_NAME'] +allSubDomains = os.environ["ALL_SUBDOMAINS"].split(",") +snapshotWebsiteBaseDomain = os.environ["SNAPSHOT_WEBSITE_DOMAIN_NAME"] -filename="tezos-snapshots.json" +filename = "tezos-snapshots.json" # Write empty top-level array to initialize json json_object = [] @@ -15,16 +15,18 @@ # Get each subdomain's base.json and combine all artifacts into 1 metadata file for subDomain in allSubDomains: - baseJsonUrl="https://"+subDomain+"."+snapshotWebsiteBaseDomain+"/base.json" - try: - with urllib.request.urlopen(baseJsonUrl) as url: - data = json.loads(url.read().decode()) - for entry in data: - json_object.append(entry) - except urllib.error.HTTPError: - continue + baseJsonUrl = ( + "https://" + subDomain + "." + snapshotWebsiteBaseDomain + "/base.json" + ) + try: + with urllib.request.urlopen(baseJsonUrl) as url: + data = json.loads(url.read().decode()) + for entry in data: + json_object.append(entry) + except urllib.error.HTTPError: + continue # Write to file -with open (filename, 'w') as json_file: - json_string = json.dumps(json_object, indent=4) - json_file.write(json_string) \ No newline at end of file +with open(filename, "w") as json_file: + json_string = json.dumps(json_object, indent=4) + json_file.write(json_string) diff --git a/snapshotEngine/getLatestSnapshotMetadata.py b/snapshotEngine/getLatestSnapshotMetadata.py index a7a39edcd..532385ebd 100644 --- a/snapshotEngine/getLatestSnapshotMetadata.py +++ b/snapshotEngine/getLatestSnapshotMetadata.py @@ -10,14 +10,14 @@ import pytz from genericpath import exists -filename='tezos-snapshots.json' +filename = "tezos-snapshots.json" if exists(filename): - print('SUCCESS tezos-snapshots.json exists locally!') - with open(filename,'r') as localJson: + print("SUCCESS tezos-snapshots.json exists locally!") + with open(filename, "r") as localJson: snapshots = json.load(localJson) else: - print('ERROR tezos-snapshots.json does not exist locally!') + print("ERROR tezos-snapshots.json does not exist locally!") # sort per network snapshots_per_network = {} @@ -25,9 +25,9 @@ # for some reason, the first page is empty, so we initialize the map with dummy data to generate a first page # The error is: # Error reading file /home/nochem/workspace/xtz-shots-website/_layouts/latest_snapshots.md/latest_snapshots.md: Not a directory @ rb_sysopen - /home/nochem/workspace/xtz-shots-website/_layouts/latest_snapshots.md/latest_snapshots.md -latest_snapshots = [{ "name": "example", "latest_snapshots" : {}}] +latest_snapshots = [{"name": "example", "latest_snapshots": {}}] -all_snapshots = [{ "name": "example", "all_snapshots" : {}}] +all_snapshots = [{"name": "example", "all_snapshots": {}}] for snapshot in snapshots: network = snapshot["chain_name"] @@ -40,19 +40,31 @@ network_snapshots = {} # Initialize date to now, and then update with build date as we iterate - last_tezos_build_datetime=datetime.now().replace(tzinfo=pytz.UTC) - for (type, mode, path) in [("tarball", "rolling", "rolling-tarball"), ("tarball", "archive", "archive-tarball"), ("tezos-snapshot", "rolling", "rolling")]: - + last_tezos_build_datetime = datetime.now().replace(tzinfo=pytz.UTC) + for type, mode, path in [ + ("tarball", "rolling", "rolling-tarball"), + ("tarball", "archive", "archive-tarball"), + ("tezos-snapshot", "rolling", "rolling"), + ]: # Parses date from tezos build of each artifact, compares to last date, updates if older, otherwise its newer for snapshot in snapshots: - matches=datefinder.find_dates(snapshot['tezos_version']) - tezos_build_datetime=list(matches)[0] + matches = datefinder.find_dates(snapshot["tezos_version"]) + tezos_build_datetime = list(matches)[0] if tezos_build_datetime < last_tezos_build_datetime: - latest_tezos_build_version=[src for time, src in datefinder.find_dates(snapshot['tezos_version'], source=True)][1] - last_tezos_build_datetime=tezos_build_datetime + latest_tezos_build_version = [ + src + for time, src in datefinder.find_dates( + snapshot["tezos_version"], source=True + ) + ][1] + last_tezos_build_datetime = tezos_build_datetime # Snapshots of type (tarball/snapshot) and history mode - typed_snapshots = [s for s in snapshots if s["artifact_type"] == type and s["history_mode"] == mode] + typed_snapshots = [ + s + for s in snapshots + if s["artifact_type"] == type and s["history_mode"] == mode + ] typed_snapshots.sort(key=lambda x: int(x["block_height"]), reverse=True) try: @@ -62,7 +74,11 @@ continue # Latest should only show oldest supported build so let's filter by the oldest supported version we found above - typed_snapshots=[t for t in typed_snapshots if latest_tezos_build_version in t['tezos_version']] + typed_snapshots = [ + t + for t in typed_snapshots + if latest_tezos_build_version in t["tezos_version"] + ] try: # Latest snapshot of type is the first item in typed_snapshots which we just filtered by the latest supported tezos build @@ -71,10 +87,24 @@ continue latest_snapshots.append( - { "name": network, "permalink": network+"/index.html", "latest_snapshots": network_latest_snapshots }) + { + "name": network, + "permalink": network + "/index.html", + "latest_snapshots": network_latest_snapshots, + } + ) all_snapshots.append( - { "name": network, "permalink": network+"/list.html", "snapshots": network_snapshots }) + { + "name": network, + "permalink": network + "/list.html", + "snapshots": network_snapshots, + } + ) Path("_data").mkdir(parents=True, exist_ok=True) -with open(f"_data/snapshot_jekyll_data.json", 'w') as f: - json.dump({"latest_snapshots": latest_snapshots, "all_snapshots": all_snapshots}, f, indent=2) +with open(f"_data/snapshot_jekyll_data.json", "w") as f: + json.dump( + {"latest_snapshots": latest_snapshots, "all_snapshots": all_snapshots}, + f, + indent=2, + ) diff --git a/utils/config-generator.py b/utils/config-generator.py index 7b0dcf62b..45cf8ba4b 100755 --- a/utils/config-generator.py +++ b/utils/config-generator.py @@ -627,7 +627,11 @@ def create_node_snapshot_config_json(history_mode): """Create this node's snapshot config""" network_name = NETWORK_CONFIG.get("chain_name") - prefer_tarballs = os.environ.get("PREFER_TARBALLS", "").lower() in ("true", "1", "t") + prefer_tarballs = os.environ.get("PREFER_TARBALLS", "").lower() in ( + "true", + "1", + "t", + ) artifact_type = "tarball" if prefer_tarballs else "tezos-snapshot" rolling_tarball_url = os.environ.get("ROLLING_TARBALL_URL") full_tarball_url = os.environ.get("FULL_TARBALL_URL") diff --git a/utils/sidecar.py b/utils/sidecar.py index 0185b2909..085b149bb 100755 --- a/utils/sidecar.py +++ b/utils/sidecar.py @@ -5,24 +5,26 @@ import datetime import logging -log = logging.getLogger('werkzeug') + +log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) application = Flask(__name__) AGE_LIMIT_IN_SECS = 600 -@application.route('/is_synced') + +@application.route("/is_synced") def sync_checker(): - ''' + """ Here we don't trust the /is_bootstrapped endpoint of octez-node. We have seen it return true when the node is in a bad state (for example, some crashed threads) Instead, we query the head block and verify timestamp is not too old. - ''' + """ try: - r = requests.get('http://127.0.0.1:8732/chains/main/blocks/head/header') + r = requests.get("http://127.0.0.1:8732/chains/main/blocks/head/header") except requests.exceptions.RequestException as e: err = "Could not connect to node, %s" % repr(e), 500 print(err) @@ -34,13 +36,20 @@ def sync_checker(): # otherwise it will never activate (activation uses rpc service) return "Chain has not been activated yet" timestamp = r.json()["timestamp"] - block_age = datetime.datetime.utcnow() - datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ') + block_age = datetime.datetime.utcnow() - datetime.datetime.strptime( + timestamp, "%Y-%m-%dT%H:%M:%SZ" + ) age_in_secs = block_age.total_seconds() if age_in_secs > AGE_LIMIT_IN_SECS: - err = "Error: Chain head is %s secs old, older than %s" % ( age_in_secs, AGE_LIMIT_IN_SECS), 500 + err = ( + "Error: Chain head is %s secs old, older than %s" + % (age_in_secs, AGE_LIMIT_IN_SECS), + 500, + ) print(err) return err return "Chain is bootstrapped" + if __name__ == "__main__": - application.run(host = "0.0.0.0", port = 31732, debug = False) + application.run(host="0.0.0.0", port=31732, debug=False)