TRD mainnet adaptations (#514)

* add network selector for tezos reward distributor

* add more missing params

* introduce dedicated script for trd

* proper quoting of env vars

* fix typo in dry_run

* save more failed jobs

* add bucket uploader

* add bucket report uploader for trd

* make sure 2 diff instances of trd can be deployed in the same ns

* manage aws secret key for upload as k8s secret

* create an export file for trd bucket upload secrets

* fix wrong path

charts/tezos-reward-distributor/scripts/bucket_upload.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+echo "Uploading TRD data to bucket"
+
+source /trd/cfg/bucket_upload_secrets
+if [ ! -z ${BUCKET_NAME} ];then
+ aws s3 cp --recursive /trd/ s3://${BUCKET_NAME}/${BAKER_NAME} --endpoint $BUCKET_ENDPOINT_URL
+fi
+sleep 10

charts/tezos-reward-distributor/scripts/bucket_upload_secrets

@@ -0,0 +1,3 @@
+export AWS_ACCESS_KEY_ID="{{ .Values.bucket_upload_secrets.access_key_id }}"
+export AWS_SECRET_ACCESS_KEY="{{ .Values.bucket_upload_secrets.secret_access_key }}"
+export AWS_DEFAULT_REGION="{{ .Values.bucket_upload_secrets.default_region }}"

charts/tezos-reward-distributor/scripts/run.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+if [ "${DRY_RUN}" == "false" ]; then
+ dry_run_arg=""
+else
+ dry_run_arg="--dry_run"
+fi
+python src/main.py \
+ -M 2 \
+ --reward_data_provider ${REWARD_DATA_PROVIDER} \
+ --node_addr_public ${TEZOS_NODE_ADDR} \
+ --node_endpoint ${TEZOS_NODE_ADDR} \
+ --base_directory /trd \
+ --signer_endpoint ${SIGNER_ADDR} \
+ --release_override ${RELEASE_OVERRIDE} \
+ --initial_cycle ${INITIAL_CYCLE} \
+ -N ${NETWORK} \
+ ${EXTRA_TRD_ARGS} \
+ ${dry_run_arg}

charts/tezos-reward-distributor/templates/config.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
- name: tezos-reward-distributor-config
+ name: {{ include "tezos-reward-distributor.fullname" . }}-config
 data:
  config.yaml: |2+
  {{ toYaml $.Values.trd_config |nindent 4}}

charts/tezos-reward-distributor/templates/cronjob.yaml

@@ -7,8 +7,9 @@ metadata:
 spec:
 
  # important to prevent launch of concurrent payout processes
- concurrencyPolicy:  Forbid
+ concurrencyPolicy: Forbid
 
+ failedJobsHistoryLimit: 100
  schedule: {{ .Values.schedule }}
  jobTemplate:
  metadata:
@@ -26,10 +27,13 @@ spec:
  volumes:
  - name: storage
  persistentVolumeClaim:
- claimName: tezos-reward-distributor-volume
+ claimName: {{ include "tezos-reward-distributor.fullname" . }}-volume
  - name: config-volume
  configMap:
- name: tezos-reward-distributor-config
+ name: {{ include "tezos-reward-distributor.fullname" . }}-config
+ - name: secret-volume
+ secret:
+ secretName: {{ include "tezos-reward-distributor.fullname" . }}-secret
  initContainers:
  # Work around a bug where fsGroup is ignored
  - name: change-ownership-container
@@ -44,18 +48,58 @@ spec:
  volumeMounts:
  - mountPath: /trd
  name: storage
+ - name: tezos-reward-distributor-cron-job
+ image: {{ .Values.images.tezos_reward_distributor }}
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /trd
+ name: storage
+ - mountPath: /trd/cfg/config.yaml
+ name: config-volume
+ subPath: config.yaml
+ command:
+ - /bin/sh
+ args:
+ - "-c"
+ - |
+{{ tpl ($.Files.Get (print "scripts/run.sh")) $ | indent 16 }}
+ env:
+ - name: REWARD_DATA_PROVIDER
+ value: "{{ .Values.reward_data_provider }}"
+ - name: TEZOS_NODE_ADDR
+ value: "{{ .Values.tezos_node_addr }}"
+ - name: SIGNER_ADDR
+ value: "{{ .Values.signer_addr }}"
+ - name: EXTRA_TRD_ARGS
+ value: "{{ .Values.extra_trd_args }}"
+ - name: NETWORK
+ value: "{{ .Values.network }}"
+ - name: RELEASE_OVERRIDE
+ value: "{{ .Values.release_override }}"
+ - name: INITIAL_CYCLE
+ value: "{{ .Values.initial_cycle }}"
+ - name: DRY_RUN
+ value: "{{ .Values.dry_run }}"
  containers:
- - name: tezos-reward-distributor-cron-job
- image: {{ .Values.images.tezos_reward_distributor }}
- imagePullPolicy: IfNotPresent
- volumeMounts:
+ - name: report-uploader
+ image: {{ .Values.tezos_k8s_images.snapshotEngine }}
+ volumeMounts:
  - mountPath: /trd
  name: storage
- - mountPath: /trd/cfg/config.yaml
- name: config-volume
- subPath: config.yaml
- command:
- - /bin/sh
- - -c
- - python src/main.py -M 2 --reward_data_provider {{ .Values.reward_data_provider }} --node_addr_public {{ .Values.tezos_node_addr }} --node_endpoint {{ .Values.tezos_node_addr }} --base_directory /trd --signer_endpoint {{ .Values.signer_addr }} {{ .Values.extra_trd_args }} -N GHOSTNET
+ - mountPath: /trd/cfg/bucket_upload_secrets
+ name: secret-volume
+ subPath: bucket_upload_secrets
+ command:
+ - /bin/sh
+ args:
+ - "-c"
+ - |
+{{ tpl ($.Files.Get (print "scripts/bucket_upload.sh")) $ | indent 16 }}
+ env:
+ - name: BUCKET_ENDPOINT_URL
+ value: "{{ .Values.bucket_upload.bucket_endpoint_url }}"
+ - name: BUCKET_NAME
+ value: "{{ .Values.bucket_upload.bucket_name }}"
+ - name: BAKER_NAME
+ value: {{ include "tezos-reward-distributor.fullname" . }}
  restartPolicy: OnFailure

charts/tezos-reward-distributor/templates/secrets.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "tezos-reward-distributor.fullname" . }}-secret
+data:
+ bucket_upload_secrets: {{ tpl (.Files.Get "scripts/bucket_upload_secrets") . | b64enc | quote }}

charts/tezos-reward-distributor/templates/volume.yaml

@@ -1,7 +1,7 @@
 apiVersion: "v1"
 kind: PersistentVolumeClaim
 metadata:
- name: tezos-reward-distributor-volume
+ name: {{ include "tezos-reward-distributor.fullname" . }}-volume
 spec:
  storageClassName:
  accessModes:

charts/tezos-reward-distributor/values.yaml

@@ -1,6 +1,12 @@
 images:
  tezos_reward_distributor: trdo/tezos-reward-distributor:latest
 
+tezos_k8s_images:
+ # snapshotEngine is needed for upload of logs to bucket
+ # since it already exists, we do not create a new container
+ # just for this task.
+ snapshotEngine: ghcr.io/oxheadalpha/tezos-k8s-snapshotengine:master
+
 # The node endpoint. It must be an archive node.
 # May start with https://
 # Default value will connect to a tezos-k8s private chain running in the same workspace, with
@@ -23,74 +29,96 @@ schedule: "0 */6 * * *"
 # Pick one of "rpc", "tzstats", "tzkt"
 reward_data_provider: "rpc"
 
+# Tezos Network. Can be MAINNET or GHOSTNET
+network: MAINNET
+
+# Release override: set to -5 to pay just finished cycle
+release_override: -5
+
+# Set initial cycle to pay rewards from. Set to -1 to start from just finished cycle.
+initial_cycle: -1
+
+# Dry-run. Set to "true" to not actually perform the payouts.
+dry_run: false
+
 extra_trd_args: "--do_not_publish_stats"
 
 # This is an example of TRD config. Customize to your needs.
 # For details, please consult TRD documentation:
 # https://tezos-reward-distributor-organization.github.io/tezos-reward-distributor/
 trd_config:
- version: 1.0
- baking_address: tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2
- payment_address: tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2
- rewards_type: actual
- service_fee: 4.5
- founders_map:
- {'tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2' : 1.0}
- owners_map:
- {'tz1eawTP2ueRCqsYZq3DebwW9PkthdCqcZa8' : 0.3,
- 'tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2' : 0.7}
- specials_map: {}
- supporters_set: {}
- min_delegation_amt: 100
- reactivate_zeroed: True
- delegator_pays_xfer_fee: True
- delegator_pays_ra_fee: True
- pay_denunciation_rewards: True
- rules_map:
- KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn3: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn0
- KT1D33n8zp1bqBkViiQtLLPLEGRW9xcqihY3: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn0
- KT1Ao8UXNJ9Dz71Wx3m8yzYNdnNQp2peqtM0: TOE
- KT1VyxJWhe9oz3v4qwTp2U6Rb17ocHGpJmW0: TOB
- KT19cJWfbDNXT4azVbgTBvtLMeqweuHH8W20: TOF
- KT1DextebDNXT4azVbgTBvtLMeqweuHH8W20: Dexter
- mindelegation: TOB
- plugins:
- enabled:
- - webhook
- email:
- smtp_user: user@example.com
- smtp_pass: horsebatterystaple2
- smtp_host: smtp.example.com
- smtp_port: 587
- smtp_tls: true
- smtp_sender: trdnotice@example.com
- smtp_recipients:
- - bob@example.com
- - alice@example.com
- telegram:
- admin_chat_ids:
- - 123456789
- payouts_chat_ids:
- - -13134455
- bot_api_key: 988877766:SKDJFLSJDFJLJSKDFJLKSDJFLKJDF
- telegram_text: >
- Rewards for cycle %CYCLE% are completed.
- We paid out %TREWARDS% tez in rewards to %NDELEGATORS% delegators.
- twitter:
- api_key: XXXXXXXX
- api_secret: ZZZZZZZZ
- access_token: YYYYYYYY
- access_secret: WWWWWWWW
- extra_tags:
- - "our_baker"
- - "tezos"
- - "rewards"
- webhook:
- endpoint: https://example.com/webhook.php
- token: Xynl6svphysd3BhjLP6IS
- discord:
- endpoint: https://discord.com/api/webhooks/9876543212345678/OmAfadfasdfasdfasdfasdfasdfasfsdf
- send_admin: False
- discord_text: >
- Rewards for cycle %CYCLE% are completed.
- We paid out %TREWARDS% tez in rewards to %NDELEGATORS% delegators.
+ # version: 1.0
+ # baking_address: tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2
+ # payment_address: tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2
+ # rewards_type: actual
+ # service_fee: 4.5
+ # founders_map:
+ # {'tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2' : 1.0}
+ # owners_map:
+ # {'tz1eawTP2ueRCqsYZq3DebwW9PkthdCqcZa8' : 0.3,
+ # 'tz1WrZpjVRpsmjv8VL3781RhyLi5JyHsGXK2' : 0.7}
+ # specials_map: {}
+ # supporters_set: {}
+ # min_delegation_amt: 100
+ # reactivate_zeroed: True
+ # delegator_pays_xfer_fee: True
+ # delegator_pays_ra_fee: True
+ # pay_denunciation_rewards: True
+ # rules_map:
+ # KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn3: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn0
+ # KT1D33n8zp1bqBkViiQtLLPLEGRW9xcqihY3: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn0
+ # KT1Ao8UXNJ9Dz71Wx3m8yzYNdnNQp2peqtM0: TOE
+ # KT1VyxJWhe9oz3v4qwTp2U6Rb17ocHGpJmW0: TOB
+ # KT19cJWfbDNXT4azVbgTBvtLMeqweuHH8W20: TOF
+ # KT1DextebDNXT4azVbgTBvtLMeqweuHH8W20: Dexter
+ # mindelegation: TOB
+ # plugins:
+ # enabled:
+ # - webhook
+ # email:
+ # smtp_user: user@example.com
+ # smtp_pass: horsebatterystaple2
+ # smtp_host: smtp.example.com
+ # smtp_port: 587
+ # smtp_tls: true
+ # smtp_sender: trdnotice@example.com
+ # smtp_recipients:
+ # - bob@example.com
+ # - alice@example.com
+ # telegram:
+ # admin_chat_ids:
+ # - 123456789
+ # payouts_chat_ids:
+ # - -13134455
+ # bot_api_key: 988877766:SKDJFLSJDFJLJSKDFJLKSDJFLKJDF
+ # telegram_text: >
+ # Rewards for cycle %CYCLE% are completed.
+ # We paid out %TREWARDS% tez in rewards to %NDELEGATORS% delegators.
+ # twitter:
+ # api_key: XXXXXXXX
+ # api_secret: ZZZZZZZZ
+ # access_token: YYYYYYYY
+ # access_secret: WWWWWWWW
+ # extra_tags:
+ # - "our_baker"
+ # - "tezos"
+ # - "rewards"
+ # webhook:
+ # endpoint: https://example.com/webhook.php
+ # token: Xynl6svphysd3BhjLP6IS
+ # discord:
+ # endpoint: https://discord.com/api/webhooks/9876543212345678/OmAfadfasdfasdfasdfasdfasdfasfsdf
+ # send_admin: False
+ # discord_text: >
+ # Rewards for cycle %CYCLE% are completed.
+ # We paid out %TREWARDS% tez in rewards to %NDELEGATORS% delegators.
+
+# optionally upload all TRD state to a bucket. This allows all data to be examined
+# when the cronjob is not running.
+bucket_upload:
+ bucket_endpoint_url:
+ bucket_name: 
+bucket_upload_secrets:
+ access_key_id:
+ default_region:
+ secret_access_key: