chore: Build Ubuntu 18 binaries

.github/workflows/ami-build-ubuntu-18.yml

@@ -0,0 +1,87 @@
+name: Build Ubuntu 18 AMI
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - '.github/workflows/ami-build-ubuntu-18.yml'
+      - 'common.vars.pkr.hcl'
+
+  workflow_run:
+    workflows: [Release AMI]
+    types:
+      - completed
+
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: [self-hosted, X64]
+    timeout-minutes: 150
+    permissions:
+      contents: write
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+
+      - name: Build AMI
+        run: |
+          GIT_SHA=${{github.sha}}
+          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=u18-${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" -var-file="legacy.vars.pkr.hcl" amazon-arm64.pkr.hcl
+
+      - name: Grab release version
+        id: process_release_version
+        run: |
+          VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common.vars.pkr.hcl)
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          
+      - name: List files
+        id: list_files
+        run: |
+          ls -la /tmp/
+
+      - name: configure aws credentials - staging
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload pg binaries to s3 staging
+        run: |
+          aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/18.04.tar.gz
+
+      - name: configure aws credentials - prod
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.PROD_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload pg binaries to s3 prod
+        run: |
+          aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/18.04.tar.gz
+
+      - name: Deregister image
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          IMAGE_IDS=$(aws ec2 describe-images --filters "Name=name,Values=builder-supabase-postgres-u18-*" --query "Images[] .ImageId" --output text)
+          for IMAGE_ID in $IMAGE_IDS; do
+            SNAPSHOTS=$(aws ec2 describe-images --image-ids $IMAGE_ID --query "Images[*].BlockDeviceMappings[*].Ebs.SnapshotId" --output text)
+
+            echo "Deregistering image $IMAGE_ID"
+            aws ec2 deregister-image --image-id $IMAGE_ID
+
+            for SNAPSHOT in $SNAPSHOTS; do
+              echo "Deleting snapshot $SNAPSHOT"
+              aws ec2 delete-snapshot --snapshot-id $SNAPSHOT
+            done
+          done
+
+      - name: Cleanup resources on build cancellation
+        if: ${{ cancelled() }}
+        run: |
+          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=u18-${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {}

.github/workflows/ami-release.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Build AMI
         run: |
           GIT_SHA=${{github.sha}}
-          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" amazon-arm64.pkr.hcl
+          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" -var "ansible_arguments="  amazon-arm64.pkr.hcl
 
       - name: Grab release version
         id: process_release_version
@@ -47,6 +47,10 @@ jobs:
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
             manifest-playbook.yml
 
+      - name: Upload pg binaries to s3 staging
+        run: |
+          aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
+
       - name: configure aws credentials - prod
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -61,6 +65,10 @@ jobs:
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
             manifest-playbook.yml
 
+      - name: Upload pg binaries to s3 prod
+        run: |
+          aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
+
       - name: Create release
         uses: softprops/action-gh-release@v1
         with:

amazon-arm64.pkr.hcl

@@ -20,7 +20,7 @@ variable "ami_regions" {
 
 variable "ansible_arguments" {
   type    = string
-  default = "--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-supabase-internal,ebssurrogate_mode='true'"
+  default = "--skip-tags install-postgrest,install-pgbouncer,install-supabase-internal"
 }
 
 variable "aws_access_key" {
@@ -239,8 +239,9 @@ build {
       "DOCKER_IMAGE=${var.docker_image}",
       "DOCKER_IMAGE_TAG=${var.docker_image_tag}"
     ]
+    use_env_var_file = true
     script = "ebssurrogate/scripts/surrogate-bootstrap.sh"
-    execute_command = "sudo -S sh -c '{{ .Vars }} {{ .Path }}'"
+    execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'"
     start_retry_timeout = "5m"
     skip_clean = true
   }
@@ -250,4 +251,10 @@ build {
     destination = "/tmp/ansible.log"
     direction = "download"
   }
+
+  provisioner "file" {
+    source = "/tmp/pg_binaries.tar.gz"
+    destination = "/tmp/pg_binaries.tar.gz"
+    direction = "download"
+  }
 }

ansible/playbook.yml

@@ -154,6 +154,8 @@
 
     - name: Run migrations
       import_tasks: tasks/setup-migrations.yml
+      tags:
+        - migrations
 
     - name: Stop Postgres Database without Systemd
       become: yes
@@ -164,3 +166,10 @@
 
     - name: Run unit tests
       import_tasks: tasks/test-image.yml
+      tags:
+        - unit-tests
+
+    - name: Collect Postgres binaries
+      import_tasks: tasks/internal/collect-pg-binaries.yml
+      tags:
+        - collect-binaries

ansible/tasks/finalize-ami.yml

@@ -27,13 +27,31 @@
     port: http
   tags:
   - install-supabase-internal 
+  when: ansible_distribution_version != "18.04"
 
 - name: UFW - Allow connections to https (443)
   ufw:
     rule: allow
     port: https
   tags:
   - install-supabase-internal 
+  when: ansible_distribution_version != "18.04"
+
+- name: UFW - Allow connections to http (80) - Ubuntu 18.04
+  ufw:
+    rule: allow
+    port: "80"
+  tags:
+  - install-supabase-internal 
+  when: ansible_distribution_version == "18.04"
+
+- name: UFW - Allow connections to https (443) - Ubuntu 18.04
+  ufw:
+    rule: allow
+    port: "443"
+  tags:
+  - install-supabase-internal 
+  when: ansible_distribution_version == "18.04"
 
 - name: UFW - Deny all other incoming traffic by default
   ufw:
@@ -65,6 +83,7 @@
         sed -i -e 's;daily;*:0/10;' /etc/systemd/system/logrotate.timer
         systemctl reenable logrotate.timer
   become: yes
+  when: ansible_distribution_version != "18.04"
 
 - name: import pgsodium_getkey script
   template:

ansible/tasks/internal/collect-pg-binaries.yml

@@ -0,0 +1,31 @@
+- name: Collect Postgres binaries - create collection directory
+  file:
+    path: /tmp/pg_binaries/{{ postgresql_major }}/
+    state: directory
+
+- name: Collect Postgres binaries - collect binaries and libraries
+  copy:
+    remote_src: yes
+    src: /usr/lib/postgresql/{{ item }}/
+    dest: /tmp/pg_binaries/{{ postgresql_major }}/{{ item }}/
+  with_items:
+    - bin
+    - lib
+
+- name: Collect Postgres binaries - collect shared files
+  copy:
+    remote_src: yes
+    src: /var/lib/postgresql/
+    dest: /tmp/pg_binaries/{{ postgresql_major }}/share/
+
+- name: Collect Postgres binaries - create tarfile
+  archive:
+    path: /tmp/pg_binaries/
+    dest: /tmp/pg_binaries.tar.gz
+    remove: yes
+
+- name: Fetch tarfile to local
+  fetch:
+    src: /tmp/pg_binaries.tar.gz
+    dest: /tmp/
+    flat: true

ansible/tasks/internal/supautils.yml

@@ -6,6 +6,16 @@
       - clang-11
     update_cache: yes
     cache_valid_time: 3600
+  when: ansible_distribution_version != "18.04"
+
+- name: supautils - download & install dependencies - Ubuntu 18.04
+  apt:
+    pkg:
+      - build-essential
+      - clang-12
+    update_cache: yes
+    cache_valid_time: 3600
+  when: ansible_distribution_version != "18.04"
 
 - name: supautils - download latest release
   get_url:

ansible/tasks/postgres-extensions/01-postgis.yml

@@ -49,7 +49,8 @@
   become: yes
 
 - name: postgis - build SFCGAL
-  make:
+  community.general.make:
+    target: all
     chdir: /tmp/SFCGAL-v{{ sfcgal_release }}
     jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes
@@ -78,7 +79,8 @@
   become: yes
 
 - name: postgis - build
-  make:
+  community.general.make:
+    target: all
     chdir: /tmp/postgis-{{ postgis_release }}
     jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes

ansible/tasks/postgres-extensions/02-pgrouting.yml

@@ -34,7 +34,8 @@
   become: yes
 
 - name: pgRouting - build
-  make:
+  community.general.make:
+    target: all
     chdir: /tmp/pgrouting-{{ pgrouting_release }}/build
     jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes

ansible/tasks/postgres-extensions/18-pgsodium.yml

@@ -53,7 +53,6 @@
 - name: pgsodium - build
   make:
     chdir: /tmp/pgsodium-{{ pgsodium_release }}
-    jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes
 
 - name: pgsodium - install

ansible/tasks/postgres-extensions/24-pgroonga.yml

@@ -33,7 +33,8 @@
   become: yes
 
 - name: groonga - build
-  make:
+  community.general.make:
+    target: all
     chdir: /tmp/groonga-{{ groonga_release }}
     jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes
@@ -59,7 +60,8 @@
   become: yes
 
 - name: pgroonga - build
-  make:
+  community.general.make:
+    target: all
     chdir: /tmp/pgroonga-{{ pgroonga_release }}
     jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes

ansible/tasks/postgres-extensions/28-pgvector.yml

@@ -5,6 +5,14 @@
     dest: /tmp/pgvector
     version: 'v{{ pgvector_release }}'
   become: yes
+
+- name: pgvector - ubuntu 18.04 compat
+  lineinfile:
+    path: /tmp/pgvector/Makefile
+    regexp: "march=native$"
+    line: "OPTFLAGS = "
+    firstmatch: true
+  when: ansible_distribution_version == "18.04"
 
 - name: pgvector - build
   make:

ansible/tasks/setup-extensions.yml

@@ -39,6 +39,8 @@
 
 - name: Install pljava
   import_tasks: tasks/postgres-extensions/12-pljava.yml
+  tags:
+    - legacy-incompatible
 
 - name: Install pg_plan_filter
   import_tasks: tasks/postgres-extensions/14-pg_plan_filter.yml
@@ -57,6 +59,8 @@
 
 - name: Install pg_graphql
   import_tasks: tasks/postgres-extensions/19-pg_graphql.yml
+  tags:
+    - legacy-incompatible
 
 - name: Install pg_stat_monitor
   import_tasks: tasks/postgres-extensions/20-pg_stat_monitor.yml

ansible/tasks/setup-nginx.yml

@@ -37,7 +37,8 @@
   become: yes
 
 - name: nginx - build
-  make:
+  community.general.make:
+    target: build
     chdir: /tmp/nginx-{{ nginx_release }}
     jobs: "{{ parallel_jobs | default(omit) }}"
   become: yes