ci: update protonet reset workflow for new infra

.github/scripts/put-all-chain-nodes-on-standby.sh

@@ -2,20 +2,23 @@
 set -x
 
 # get all the node's ec2 instance ids for the specified chain id
-chain_node_instance_ids=$(aws ec2 describe-instances --filters "Name=tag:KavaChainId,Values=$CHAIN_ID" | jq -r '[.Reservations | .[] | .Instances | .[] | .InstanceId] | join(" ")')
+chain_node_instance_ids=$(aws ec2 describe-instances --filters "Name=tag:ChainId,Values=$CHAIN_ID" | jq -r '[.Reservations | .[] | .Instances | .[] | .InstanceId] | join(" ")')
+
+# get autoscaling group details about instances
+asg_info=$(aws autoscaling describe-auto-scaling-instances --instance-ids $chain_node_instance_ids | jq '[.AutoScalingInstances[] | {id: .InstanceId, asg: .AutoScalingGroupName, state: .LifecycleState}]')
+
+echo "$asg_info" | jq -c '.[]' | while read instance; do
+ chain_node_instance_id=$(echo $instance | jq -r '.id')
+ autoscaling_group_name=$(echo $instance | jq -r '.asg')
+ autoscaling_group_state=$(echo $instance | jq -r '.state')
+
+ echo "$instance"
 
-for chain_node_instance_id in ${chain_node_instance_ids}
-do
- autoscaling_group_state=$(aws autoscaling describe-auto-scaling-instances --instance-ids "$chain_node_instance_id" | jq -r '[.AutoScalingInstances | .[].LifecycleState] | join(" ")')
  # Possible states: https://docs.aws.amazon.com/autoscaling/ec2/userguide/ec2-auto-scaling-lifecycle.html
  case "$autoscaling_group_state" in
  InService)
- # place the nodes on standby so they won't get terminated
- # by the autoscaling group during the time
- # they are offline for a deploy / upgrade
- autoscaling_group_name=$(aws autoscaling describe-auto-scaling-instances --instance-ids "$chain_node_instance_id" | jq -r '[.AutoScalingInstances | .[].AutoScalingGroupName] | join(" ")')
-
- aws autoscaling enter-standby \
+ # put each instance in standby mode
+ aws autoscaling enter-standby \
  --instance-ids "$chain_node_instance_id" \
  --auto-scaling-group-name "$autoscaling_group_name" \
  --should-decrement-desired-capacity

.github/workflows/cd-protonet.yml

@@ -1,41 +1,26 @@
 name: Continuous Deployment (Protonet)
-## run after every successful CI job of new commits to the master branch
-#on:
-# workflow_run:
-# workflows: [Continuous Integration (Kava Master)]
-# types:
-# - completed
 
-jobs:
- # in order:
- # enter standby (prevents autoscaling group from killing node during deploy)
- # stop kava
- # take ebs + zfs snapshots
- # download updated binary and genesis
- # reset application database state (only done on internal testnet)
- reset-chain-to-zero-state:
- # only start cd pipeline if last ci run was successful
- if: ${{ github.event.workflow_run.conclusion == 'success' }}
- uses: ./.github/workflows/cd-reset-internal-testnet.yml
- with:
- aws-region: us-east-1
- chain-id: proto_2221-17000
- ssm-document-name: kava-testnet-internal-node-update
- playbook-name: reset-protonet-playbook.yml
- playbook-infrastructure-branch: master
- secrets: inherit
+## run after every successful CI job of new commits to the master branch, or on manual dispatch
+on:
+ workflow_run:
+ workflows: [Continuous Integration Kava (Master)]
+ types:
+ - completed
+ workflow_dispatch:
 
- # start kava with new binary and genesis state on api, peer and seed nodes, place nodes in service once they start and are synched to live
- start-chain-api:
- uses: ./.github/workflows/cd-start-chain.yml
+jobs:
+ reset-and-restart-chain:
+ # only start cd pipeline if last ci run was successful (or it this is being manually dispatched)
+ if: |
+ github.event_name == 'workflow_dispatch' ||
+ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
+ uses: ./.github/workflows/cd-reset-network.yml
  with:
  aws-region: us-east-1
  chain-id: proto_2221-17000
- ssm-document-name: kava-testnet-internal-node-update
- playbook-name: start-chain-api-playbook.yml
+ ssm-document-name: kava-protonet-api-instance-update
  playbook-infrastructure-branch: master
  secrets: inherit
- needs: [reset-chain-to-zero-state]
 
  # setup test and development accounts and balances, deploy contracts by calling the chain's api
  seed-chain-state:
@@ -46,9 +31,10 @@ jobs:
  seed-script-filename: seed-protonet.sh
  erc20-deployer-network-name: protonet
  genesis_validator_addresses: "kavavaloper14w4avgdvqrlpww6l5dhgj4egfn6ln7gmtp7r2m"
- kava_version_filepath:  ./ci/env/kava-protonet/KAVA.VERSION
+ kava_version_filepath: ./ci/env/kava-protonet/KAVA.VERSION
  secrets: inherit
- needs: [start-chain-api]
+ needs: [reset-and-restart-chain]
+
  post-pipeline-metrics:
  uses: ./.github/workflows/metric-pipeline.yml
  if: always() # always run so we metric failures and successes

.github/workflows/cd-reset-network.yml

@@ -0,0 +1,99 @@
+# for use with ephemeral testnets like internal testnet
+
+name: Reset Ephemeral Network
+on:
+ workflow_call:
+ inputs:
+ chain-id:
+ required: true
+ type: string
+ aws-region:
+ required: true
+ type: string
+ ssm-document-name:
+ required: true
+ type: string
+ playbook-infrastructure-branch:
+ required: true
+ type: string
+ secrets:
+ CI_AWS_KEY_ID:
+ required: true
+ CI_AWS_KEY_SECRET:
+ required: true
+ KAVA_PRIVATE_GITHUB_ACCESS_TOKEN:
+ required: true
+jobs:
+ reset-ephemeral-network:
+ runs-on: ubuntu-latest
+ steps:
+ - name: checkout infrastructure repo
+ uses: actions/checkout@v4
+ with:
+ repository: Kava-Labs/infrastructure
+ token: ${{ secrets.KAVA_PRIVATE_GITHUB_ACCESS_TOKEN }}
+ path: infrastructure
+ ref: master
+
+ - name: Set up Go with kava-node-updater cache
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: infrastructure/cli/kava-node-updater/go.mod
+ cache-dependency-path: infrastructure/cli/kava-node-updater/go.mod
+
+ - name: build kava node updater
+ run: cd infrastructure/cli/kava-node-updater && make install && cd ../../../
+
+ - name: stop chain services & reset data on all chain nodes
+ run: |
+ echo $AWS_REGION $CHAIN_ID $SSM_DOCUMENT_NAME $AWS_SDK_LOAD_CONFIG $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+ env:
+ SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+ CHAIN_ID: ${{ inputs.chain-id }}
+ AWS_SDK_LOAD_CONFIG: 1
+ AWS_REGION: ${{ inputs.aws-region }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+ PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
+
+ - name: stop chain services & reset data on all chain nodes
+ run: |
+ kava-node-updater \
+ --debug \
+ --max-retries=2 \
+ --aws-ssm-document-name=$SSM_DOCUMENT_NAME \
+ --infrastructure-git-pointer=$PLAYBOOK_INFRASTRUCTURE_BRANCH \
+ --chain-id=$CHAIN_ID \
+ --max-upgrade-batch-size=0 \
+ --wait-for-node-sync-after-upgrade=false \
+ --update-playbook-filename=reset-chain-data.yml
+ env:
+ SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+ CHAIN_ID: ${{ inputs.chain-id }}
+ AWS_SDK_LOAD_CONFIG: 1
+ AWS_REGION: ${{ inputs.aws-region }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+ PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
+
+ # NOTE: we wait for the reset to finish before restarting the nodes to prevent nodes from
+ # re-syncing the previous chain state.
+ - name: reconfigure & restart chain services on all chain nodes
+ run: |
+ kava-node-updater \
+ --debug \
+ --max-retries=2 \
+ --aws-ssm-document-name=$SSM_DOCUMENT_NAME \
+ --infrastructure-git-pointer=$PLAYBOOK_INFRASTRUCTURE_BRANCH \
+ --chain-id=$CHAIN_ID \
+ --max-upgrade-batch-size=0 \
+ --wait-for-node-sync-after-upgrade=false \
+ --update-playbook-filename=playbook.yml
+ env:
+ SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+ CHAIN_ID: ${{ inputs.chain-id }}
+ AWS_REGION: ${{ inputs.aws-region }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+ AWS_SDK_LOAD_CONFIG: 1
+ PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}