[M] Deploy Ten Gateway Backend ( dev-testnet ) #507
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Deploys TEN Gateway on Azure for Testnet | |
| # Builds the TEN Gateway image, pushes the image to dockerhub and starts the TEN Gateway on Azure VM | |
| # This action requires the following environment variables to be set: | |
| # - DOCKER_BUILD_TAG_GATEWAY | |
| # - AZURE_DEPLOY_GROUP_GATEWAY | |
| # - L2_RPC_URL_VALIDATOR | |
| # - GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME | |
| # - GATEWAY_RATE_LIMIT_WINDOW | |
| # - GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER | |
| # If we are deploying to a non primary instance all those variables should be prefixed with the instance name | |
| # example: DEXYNTH_DOCKER_BUILD_TAG_GATEWAY | |
| name: '[M] Deploy Ten Gateway Backend' | |
| run-name: '[M] Deploy Ten Gateway Backend ( ${{ github.event.inputs.testnet_type }} )' | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| testnet_type: | |
| description: "Environment" | |
| required: true | |
| default: "dev-testnet" | |
| type: choice | |
| options: | |
| - "dev-testnet" | |
| - "uat-testnet" | |
| - "sepolia-testnet" | |
| instance_type: | |
| description: "Instance" | |
| required: true | |
| default: "primary" | |
| type: choice | |
| options: | |
| - "primary" | |
| - "DEXYNTH" | |
| jobs: | |
| validate-inputs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Check if deployment is allowed" | |
| run: | | |
| if [[ "${{ github.event.inputs.instance_type }}" == "DEXYNTH" && "${{ github.event.inputs.testnet_type }}" != "sepolia-testnet" ]]; then | |
| echo "Error: Dexynth can only be deployed to sepolia-testnet." | |
| exit 1 | |
| fi | |
| build-and-deploy: | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: ${{ github.event.inputs.testnet_type }} | |
| steps: | |
| - name: "Set up environment variables" | |
| id: setup_env | |
| run: | | |
| INSTANCE_SUFFIX="" | |
| INSTANCE_PREFIX="" | |
| if [[ "${{ github.event.inputs.instance_type }}" != "primary" ]]; then | |
| INSTANCE_SUFFIX="_${{ github.event.inputs.instance_type }}" | |
| INSTANCE_SUFFIX2="-${{ github.event.inputs.instance_type }}" | |
| INSTANCE_PREFIX="${{ github.event.inputs.instance_type }}_" | |
| fi | |
| echo "INSTANCE_SUFFIX=$INSTANCE_SUFFIX" >> $GITHUB_ENV | |
| echo "INSTANCE_PREFIX=$INSTANCE_PREFIX" >> $GITHUB_ENV | |
| # Set infrastructure variables | |
| PUBLIC_IP="${{ github.event.inputs.testnet_type }}-OG-static${INSTANCE_SUFFIX2,,}" | |
| DNS_NAME="obscurogateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX2,,}" | |
| VM_NAME="${{ github.event.inputs.testnet_type }}-OG-${{ github.run_number }}${INSTANCE_SUFFIX}" | |
| DEPLOY_GROUP="ObscuroGateway-${{ github.event.inputs.testnet_type }}-${{ github.run_number }}${INSTANCE_SUFFIX}" | |
| VNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01VNET${INSTANCE_SUFFIX}" | |
| SUBNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01Subnet${INSTANCE_SUFFIX}" | |
| echo "PUBLIC_IP=$PUBLIC_IP" >> $GITHUB_ENV | |
| echo "DNS_NAME=$DNS_NAME" >> $GITHUB_ENV | |
| echo "VM_NAME=$VM_NAME" >> $GITHUB_ENV | |
| echo "DEPLOY_GROUP=$DEPLOY_GROUP" >> $GITHUB_ENV | |
| echo "VNET_NAME=$VNET_NAME" >> $GITHUB_ENV | |
| echo "SUBNET_NAME=$SUBNET_NAME" >> $GITHUB_ENV | |
| # Set instance-specific variables | |
| declare -a VAR_NAMES=( | |
| "DOCKER_BUILD_TAG_GATEWAY" | |
| "AZURE_DEPLOY_GROUP_GATEWAY" | |
| "L2_RPC_URL_VALIDATOR" | |
| "GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME" | |
| "GATEWAY_RATE_LIMIT_WINDOW" | |
| "GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER" | |
| ) | |
| for VAR_NAME in "${VAR_NAMES[@]}"; do | |
| FULL_VAR_NAME="${INSTANCE_PREFIX}${VAR_NAME}" | |
| VAR_VALUE=$(jq -r --arg key "$FULL_VAR_NAME" '.[$key] // empty' <<< '${{ toJson(vars) }}') | |
| if [[ -n "$VAR_VALUE" ]]; then | |
| echo "${VAR_NAME}=${VAR_VALUE}" >> $GITHUB_ENV | |
| else | |
| echo "Warning: ${FULL_VAR_NAME} not found in vars" >&2 | |
| fi | |
| done | |
| - name: "Print environment variables" | |
| # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference | |
| run: | | |
| echo "INSTANCE_SUFFIX: $INSTANCE_SUFFIX" | |
| echo "INSTANCE_PREFIX: $INSTANCE_PREFIX" | |
| echo "PUBLIC_IP: $PUBLIC_IP" | |
| echo "DNS_NAME: $DNS_NAME" | |
| echo "VM_NAME: $VM_NAME" | |
| echo "DEPLOY_GROUP: $DEPLOY_GROUP" | |
| echo "VNET_NAME: $VNET_NAME" | |
| echo "SUBNET_NAME: $SUBNET_NAME" | |
| echo "DOCKER_BUILD_TAG_GATEWAY: $DOCKER_BUILD_TAG_GATEWAY" | |
| echo "AZURE_DEPLOY_GROUP_GATEWAY: $AZURE_DEPLOY_GROUP_GATEWAY" | |
| echo "L2_RPC_URL_VALIDATOR: $L2_RPC_URL_VALIDATOR" | |
| echo "GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME: $GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME" | |
| echo "GATEWAY_RATE_LIMIT_WINDOW: $GATEWAY_RATE_LIMIT_WINDOW" | |
| echo "GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER: $GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER" | |
| - name: "Print GitHub variables" | |
| # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference | |
| run: | | |
| echo "GitHub Variables = ${{ toJSON(vars) }}" | |
| - uses: actions/checkout@v4 | |
| - name: "Extract branch name" | |
| shell: bash | |
| run: | | |
| echo "Branch Name: ${GITHUB_REF_NAME}" | |
| echo "BRANCH_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV | |
| - name: "Set up Docker" | |
| uses: docker/setup-buildx-action@v1 | |
| - name: "Login to Azure docker registry" | |
| uses: azure/docker-login@v1 | |
| with: | |
| login-server: testnetobscuronet.azurecr.io | |
| username: testnetobscuronet | |
| password: ${{ secrets.REGISTRY_PASSWORD }} | |
| - name: "Login via Azure CLI" | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| DOCKER_BUILDKIT=1 docker build --build-arg TESTNET_TYPE=${{ github.event.inputs.testnet_type }} -t ${{ env.DOCKER_BUILD_TAG_GATEWAY }} -f ./tools/walletextension/enclave.Dockerfile . | |
| docker push ${{ env.DOCKER_BUILD_TAG_GATEWAY }} | |
| # This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk) | |
| - name: "Delete deployed VMs" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| $(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true | |
| # This will clean up any lingering dependencies - might fail if there are no resources to cleanup | |
| - name: "Delete VMs dependencies" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| $(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true | |
| - name: "Ensure VM Static Public IP Exists" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" || az network public-ip create -g Testnet -n "${{ env.PUBLIC_IP }}" --allocation-method Static --sku Standard | |
| - name: "Assign/Update DNS Name for Public IP" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| existing_dns_name=$(az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" --query dnsSettings.domainNameLabel -o tsv) | |
| if [ -z "$existing_dns_name" ]; then | |
| az network public-ip update -g Testnet -n "${{ env.PUBLIC_IP }}" --dns-name "${{ env.DNS_NAME }}" | |
| fi | |
| - name: "Create VM for Gateway node on Azure" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| az vm create -g Testnet -n "${{ env.VM_NAME }}" \ | |
| --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \ | |
| --public-ip-address "${{ env.PUBLIC_IP }}" \ | |
| --tags deploygroup="${{ env.DEPLOY_GROUP }}" ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \ | |
| --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \ | |
| --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \ | |
| --authentication-type password | |
| - name: "Open TEN node-${{ matrix.host_id }} ports on Azure" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81 | |
| # To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready | |
| - name: "Allow time for VM initialization" | |
| shell: bash | |
| run: sleep 30 | |
| - name: "Start TEN Gateway on Azure" | |
| uses: azure/CLI@v1 | |
| with: | |
| inlineScript: | | |
| az vm run-command invoke -g Testnet -n "${{ env.VM_NAME }}" \ | |
| --command-id RunShellScript \ | |
| --scripts ' | |
| set -e | |
| mkdir -p /home/obscuro | |
| # Wait for dpkg lock to be released | |
| while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do | |
| echo "Waiting for dpkg lock to be released..." | |
| sleep 1 | |
| done | |
| # Proceed with package installations | |
| sudo apt-get update | |
| sudo apt-get install -y gcc | |
| sudo snap refresh && sudo snap install --channel=1.18 go --classic | |
| # Wait again before running get-docker.sh | |
| while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do | |
| echo "Waiting for dpkg lock to be released before installing Docker..." | |
| sleep 1 | |
| done | |
| curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh | |
| git clone --depth 1 -b "${{ env.BRANCH_NAME }}" https://github.com/ten-protocol/go-ten.git /home/obscuro/go-obscuro | |
| docker network create --driver bridge node_network || true | |
| cd /home/obscuro/go-obscuro/ | |
| # Promtail Integration Start | |
| mkdir -p /home/obscuro/promtail | |
| cat <<EOF > /home/obscuro/promtail/promtail-config.yaml | |
| server: | |
| http_listen_port: 9080 | |
| grpc_listen_port: 0 | |
| positions: | |
| filename: /tmp/positions.yaml | |
| clients: | |
| - url: "${{ vars.METRICS_URI }}" | |
| batchwait: 3s | |
| batchsize: 1048576 | |
| tls_config: | |
| insecure_skip_verify: true | |
| basic_auth: | |
| username: "${{ secrets.LOKI_USER }}" | |
| password: "${{ secrets.LOKI_PASSWORD }}" | |
| scrape_configs: | |
| - job_name: flog_scrape | |
| docker_sd_configs: | |
| - host: unix:///var/run/docker.sock | |
| refresh_interval: 5s | |
| relabel_configs: | |
| - source_labels: ["__meta_docker_container_name"] | |
| regex: "/(.*)" | |
| target_label: "container" | |
| - source_labels: ["__meta_docker_container_log_stream"] | |
| target_label: "logstream" | |
| - source_labels: ["__meta_docker_container_label_logging_jobname"] | |
| target_label: "job" | |
| - replacement: "${{ env.VM_NAME }}" | |
| target_label: "node_name" | |
| EOF | |
| docker run -d --name promtail \ | |
| --network node_network \ | |
| -e HOSTNAME="${{ env.VM_NAME }}" \ | |
| -v /var/log:/var/log \ | |
| -v /home/obscuro/promtail:/etc/promtail \ | |
| -v /var/lib/docker/containers:/var/lib/docker/containers:ro \ | |
| -v /var/run/docker.sock:/var/run/docker.sock \ | |
| grafana/promtail:latest \ | |
| -config.file=/etc/promtail/promtail-config.yaml -config.expand-env=true | |
| # Promtail Integration End | |
| # Start Ten Gateway Container | |
| docker run -d -p 80:80 -p 81:81 --name "${{ env.VM_NAME }}" \ | |
| --device /dev/sgx_enclave --device /dev/sgx_provision \ | |
| -e OBSCURO_GATEWAY_VERSION="${{ github.run_number }}-${{ github.sha }}" \ | |
| -e OE_SIMULATION=0 \ | |
| "${{ env.DOCKER_BUILD_TAG_GATEWAY }}" \ | |
| ego run /home/ten/go-ten/tools/walletextension/main/main \ | |
| -host=0.0.0.0 -port=80 -portWS=81 -nodeHost="${{ env.L2_RPC_URL_VALIDATOR }}" -verbose=true \ | |
| -logPath=sys_out -dbType=cosmosDB -dbConnectionURL="${{ secrets.COSMOS_DB_CONNECTION_STRING }}" \ | |
| -rateLimitUserComputeTime="${{ env.GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME }}" -rateLimitWindow="${{ env.GATEWAY_RATE_LIMIT_WINDOW }}" -maxConcurrentRequestsPerUser="${{ env.GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER }}" ' |