Terraform + Ansible Github Actions (#1)

Specifically we have added support for the following:
* CD of Terraform IaC deployment via Github Actions
* CD of ansible configuration provisioning via GH Actions
* Update pending PR with comments stating the outcome of the terraform
and ansible deployment tasks
* Dynamic ansible inventory file generation using terraform as source
* Provisioning of secrets and host keys dynamically
* Support for deploying clickhouse to AWS via terraform and configured
via ansible
* Support for doing CD of oonidataapi with AWS ECS using terraform

---------

Co-authored-by: OONI Github Actions Bot <nothuman@ooni.org>
Co-authored-by: Simone Basso <bassosimone@gmail.com>

.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.github/workflows/check_deploy.yml

@@ -0,0 +1,219 @@
+# For docs on this see:
+# * https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+# * https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+# * https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=synchronize#pull_request
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ types:
+ - opened
+ - synchronize # when commits are pushed to the PR
+ - reopened
+ - edited # title or body of a pull request was edited, or the base branch of a pull request was changed
+
+env:
+ tf_actions_working_dir: "./tf/environments/production"
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+ ansible:
+ needs: terraform
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: ${{ env.tf_actions_working_dir }}/ansible
+ permissions:
+ pull-requests: write
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Ansible
+ run: |
+ sudo apt update
+ sudo apt install software-properties-common
+ sudo add-apt-repository --yes --update ppa:ansible/ansible-9
+ sudo apt install -y ansible
+
+ - name: Write devops ssh key to .ssh
+ run: |
+ mkdir -p ~/.ssh/
+ chmod 700 ~/.ssh/
+ echo "${{ secrets.AWS_SSH_KEY }}" > ~/.ssh/ooni-devops-prod.pem
+ chmod 600 ~/.ssh/ooni-devops-prod.pem
+
+ - name: Run Ansible Playbook
+ id: playbook
+ env:
+ ANSIBLE_SSH_ARGS: "-o UserKnownHostsFile=known_hosts"
+ run: |
+ echo "ansible_playbook<<EOF" >> "$GITHUB_OUTPUT"
+ echo "\$ ansible-playbook playbook.yml --check --diff -i inventory.ini" >> "$GITHUB_OUTPUT"
+ ansible-playbook playbook.yml --check --diff -i inventory.ini --key-file ~/.ssh/ooni-devops-prod.pem 2>&1 | tee -a "$GITHUB_OUTPUT"
+ echo "EOF" >> "$GITHUB_OUTPUT"
+ continue-on-error: true
+
+ # This can be uncommmented to make it possible to ssh into the container to debug the run
+ #- name: Setup tmate session
+ # uses: mxschmitt/action-tmate@v3
+
+ - uses: actions/github-script@v6
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const commentTitle = "Ansible Run Output";
+ const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`;
+ const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/);
+ const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : '';
+
+ const commentBody = `
+ #### Ansible Playbook Recap 🔍
+
+ \`\`\`\n
+ ${ansiblePlaybookRecap}
+ \`\`\`
+
+ #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\`
+
+ <details><summary>Show Execution</summary>
+
+ \`\`\`\n
+ ${ansiblePlaybookOutput}
+ \`\`\`
+
+ </details>
+
+ | | |
+ |-------------------|------------------------------------|
+ | Pusher | @${{ github.actor }} |
+ | Action | ${{ github.event_name }} |
+ | Working Directory | ${{ env.tf_actions_working_dir }} |
+ | Workflow | ${{ github.workflow }} |
+ | Last updated | ${(new Date()).toUTCString()} |
+ `;
+
+ // Call the script to write the comment
+ const script = require('./scripts/ghactions/comment-on-pr.js');
+ await script({github, context, core, commentTitle, commentBody});
+
+ terraform:
+ runs-on: ubuntu-latest
+ if: ${{ !startsWith(github.event.head_commit.message, 'skip-terraform:') }}
+ defaults:
+ run:
+ working-directory: ${{ env.tf_actions_working_dir }}
+ permissions:
+ contents: write
+ pull-requests: write
+ env:
+ TF_VAR_aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ TF_VAR_aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Terraform
+ run: |
+ wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+ echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+ | sudo tee /etc/apt/sources.list.d/hashicorp.list
+ sudo apt update && sudo apt install terraform
+
+ - name: Terraform fmt
+ id: fmt
+ run: terraform fmt -check
+ continue-on-error: true
+
+ - name: Terraform Init
+ id: init
+ run: terraform init
+
+ - name: Terraform Validate
+ id: validate
+ run: |
+ echo "terraform_validate<<EOF" >> "$GITHUB_OUTPUT"
+ echo "\$ terraform validate" >> "$GITHUB_OUTPUT"
+ terraform validate -no-color | tee -a "$GITHUB_OUTPUT"
+ echo "EOF" >> "$GITHUB_OUTPUT"
+
+ - name: Terraform Plan
+ id: plan
+ run: |
+ echo "terraform_plan<<EOF" >> "$GITHUB_OUTPUT"
+ echo "\$ terraform plan" >> "$GITHUB_OUTPUT"
+ terraform plan -no-color | tee -a "$GITHUB_OUTPUT"
+ echo "EOF" >> "$GITHUB_OUTPUT"
+ continue-on-error: true
+
+ # Temporarily disabled, probably should be moved to a deploy action with stricter checks
+ #- name: Terraform Apply
+ # id: apply
+ # run: |
+ # echo "terraform_apply<<EOF" >> "$GITHUB_OUTPUT"
+ # echo "\$ terraform apply -auto-approve" >> "$GITHUB_OUTPUT"
+ # terraform apply -auto-approve -no-color | tee -a "$GITHUB_OUTPUT"
+ # echo "EOF" >> "$GITHUB_OUTPUT"
+ # continue-on-error: true
+
+ - uses: actions/github-script@v6
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const terraformPlanOutput = `${{ steps.plan.outputs.terraform_plan }}`;
+ const terraformApplyOutput = `${{ steps.apply.outputs.terraform_apply }}`;
+
+ const terraformPlanPlanLine = terraformPlanOutput.split('\n').find(line => line.startsWith('Plan:'));
+ const terraformApplyPlanLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Plan:'));
+ const terraformApplyApplyLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Apply complete!'));
+
+ const commentTitle = "Terraform Run Output";
+ const commentBody = `
+ #### Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+ #### Initialization ⚙️\`${{ steps.init.outcome }}\`
+ #### Validation 🤖\`${{ steps.validate.outcome }}\`
+ <details><summary>Validation Output</summary>
+
+ \`\`\`\n
+ ${{ steps.validate.outputs.terraform_validate }}
+ \`\`\`
+
+ </details>
+
+ #### Plan 📖\`${{ steps.plan.outcome }}\`
+ * **${terraformPlanPlanLine}**
+
+ <details><summary>Show Plan</summary>
+
+ \`\`\`\n
+ ${terraformPlanOutput}
+ \`\`\`
+
+ </details>
+
+ #### Apply 📖\`${{ steps.apply.outcome }}\`
+ * **${terraformApplyPlanLine}**
+ * **${terraformApplyApplyLine}**
+
+ <details><summary>Show Apply</summary>
+
+ \`\`\`\n
+ ${terraformApplyOutput}
+ \`\`\`
+
+ </details>
+
+ | | |
+ |-------------------|------------------------------------|
+ | Pusher | @${{ github.actor }} |
+ | Action | ${{ github.event_name }} |
+ | Working Directory | ${{ env.tf_actions_working_dir }} |
+ | Workflow | ${{ github.workflow }} |
+ | Last updated | ${ (new Date()).toUTCString() } |
+ `;
+
+ // Call the script to write the comment
+ const script = require('./scripts/ghactions/comment-on-pr.js');
+ await script({github, context, core, commentTitle, commentBody});

.gitignore

@@ -1 +1,16 @@
-.terraform
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Ignore override files as they are usually used to override resources locally
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json

scripts/dump-tables-ch.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euxo pipefail
+# This is to be run manually on the clickhouse host to dump schemas and table dumps
+# You may want to make some tweaks to the dumping rules in order avoid dumping
+# too much data (eg. fastpath)
+# You should then scp the data over to the target host manually, by running:
+# $ scp * clickhouse-instance2:/var/lib/clickhouse/ooni-dumps/
+TABLES=(
+"fastpath"
+"jsonl"
+"url_priorities"
+"citizenlab"
+"citizenlab_flip"
+"test_groups"
+"accounts"
+"session_expunge"
+"msmt_feedback"
+"fingerprints_dns"
+"fingerprints_http"
+"asnmeta"
+"counters_test_list"
+"counters_asn_test_list"
+"incidents"
+"oonirun"
+)
+
+dump_dir="./dumps"
+current_date=$(date +%Y%m%d)
+
+# Directory to store the dumps
+mkdir -p "$dump_dir"
+
+# Iterate over each table
+for table in "${TABLES[@]}"; do
+ # Define file names for schema and data dump
+ schema_file="${dump_dir}/${current_date}-${table}_schema.sql"
+ data_file="${dump_dir}/${current_date}-${table}_dump.clickhouse"
+
+ # Dump the table schema
+ echo "[+] dumping schema $schema_file"
+ clickhouse-client --query="SHOW CREATE TABLE ${table} FORMAT TabSeparatedRaw" > "$schema_file"
+
+ # Dump the table data in ClickHouse native format
+ echo "[+] dumping table data $data_file"
+ clickhouse-client --query="SELECT * FROM ${table} INTO OUTFILE '${data_file}' FORMAT Native"
+done

scripts/ghactions/comment-on-pr.js

@@ -0,0 +1,44 @@
+module.exports = async ({
+ github,
+ context,
+ core,
+ commentTitle,
+ commentBody,
+}) => {
+ const body = `## ${commentTitle} 🤖
+ ${commentBody}
+ `;
+
+ const prNumber = context.payload.pull_request.number;
+
+ if (prNumber) {
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: prNumber,
+ });
+ const existingBotComment = comments.find((comment) => {
+ return (
+ comment.user.type === "Bot" &&
+ comment.body.includes(`## ${commentTitle}`)
+ );
+ });
+
+ if (existingBotComment) {
+ await github.rest.issues.updateComment({
+ issue_number: prNumber,
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: existingBotComment.id,
+ body,
+ });
+ } else {
+ await github.rest.issues.createComment({
+ issue_number: prNumber,
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body,
+ });
+ }
+ }
+};