ci: production deployment automation

.circleci/config.yml

@@ -292,6 +292,21 @@ jobs:
     executor: image-ubuntu
     environment:
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: "sparse"
+    parameters:
+      aws-access-key-id:
+        description: "AWS cli access key id"
+        type: env_var_name
+      aws-secret-access-key:
+        description: "AWS cli secret access key"
+        type: env_var_name
+      public-registry:
+        description: "Use public registry"
+        type: boolean
+        default: true
+      production:
+        description: "Push and deploy to production"
+        type: boolean
+        default: false
     steps:
       - checkout
       - run: 
@@ -300,16 +315,39 @@ jobs:
             echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
       - restore-buildx-cache
       - aws-ecr/ecr-login:
-          aws-access-key-id: AWS_ACCESS_KEY_ID
-          aws-secret-access-key: AWS_SECRET_ACCESS_KEY
-          public-registry: true
+          aws-access-key-id: << parameters.aws-access-key-id >>
+          aws-secret-access-key: << parameters.aws-secret-access-key >>
+          public-registry: << parameters.public-registry >>
       - run:
           name: Make and push images
           command: |
-            PUSH=true PROD=false PLATFORMS=linux/amd64 TAG=$TAG make images
+            PUSH=true PROD=<< parameters.production >> PLATFORMS=linux/amd64 TAG=$TAG make images
       - save-buildx-cache
   deploy-images:
     executor: image-ubuntu
+    parameters:
+      ssh-fingerprint:
+        description: "SSH-key fingerprint"
+        type: string
+        default: 6f:b3:35:dc:2d:f4:3c:e4:d1:4a:7c:a5:70:e3:b3:78
+      ssh-config-script:
+        description: "SSH config script to use."
+        type: string
+        default: unstable-ssh-config.sh
+      ssh-host:
+        description: "Shuttle SSH host"
+        type: string
+        default: shuttle.internal
+      postgres-password:
+        description: "Shuttle shared postgres password"
+        type: string
+      mongodb-password:
+        description: "Shuttle shared mongodb password"
+        type: string
+      production:
+        description: "Push and deploy to production"
+        type: boolean
+        default: false
     steps:
     - checkout
     - run: 
@@ -318,22 +356,23 @@ jobs:
           echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
     - add_ssh_keys:
         fingerprints:
-          - "6f:b3:35:dc:2d:f4:3c:e4:d1:4a:7c:a5:70:e3:b3:78"
+          - << parameters.ssh-fingerprint >>
     - run:
         name: Generate ssh config
-        command: ./.circleci/unstable-ssh-config.sh
+        command: ./.circleci/<< parameters.ssh-config-script >>
     - run:
-        name: Deploy to unstable
+        name: Deploy images
         command: |
-          DOCKER_HOST=ssh://ec2-user@master.shuttle.internal USE_TLS=enable PROD=false DD_API_KEY=$DD_API_KEY \
-          POSTGRES_PASSWORD=$DEV_POSTGRES_PASSWORD \
-          MONGO_INITDB_ROOT_PASSWORD=$DEV_MONGO_INITDB_ROOT_PASSWORD \
+          DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \
+          POSTGRES_PASSWORD=<< parameters.postgres-password >> \
+          MONGO_INITDB_ROOT_PASSWORD=<< parameters.mongodb-password >> \
           TAG=$TAG \
           make deploy
     - run:
         name: Pull new deployer image
         command: |
-          ssh ec2-user@controller.shuttle.internal "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG"
+          [[ << parameters.production >> == true ]] && ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-prod/deployer:$TAG" || \
+          ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG"
   build-binaries-linux:
     machine:
       image: << parameters.image >>
@@ -553,6 +592,19 @@ jobs:
       # - run:
       #     name: Test Docker
       #     command: ./.circleci/qa-docker.ps1
+  publish-crates:
+    parameters:
+      path:
+        description: Crate to publish
+        type: string
+    executor: docker-rust
+    resource_class: medium
+    steps:
+      - checkout
+      - run:
+          name: Crate publishing in order
+          command: |
+            cargo publish --manifest-path << parameters.path >>/Cargo.toml
 
 workflows:
   ci:
@@ -607,7 +659,7 @@ workflows:
                   "shuttle-proto",
                   "shuttle-provisioner",
                   "shuttle-runtime",
-                  "shuttle-service",
+                  "shuttle-service"
                 ]
       - e2e-test:
           requires:
@@ -623,14 +675,21 @@ workflows:
             branches:
               only: main
       - build-and-push:
+          name: build-and-push-unstable
+          aws-access-key-id: AWS_ACCESS_KEY_ID
+          aws-secret-access-key: AWS_SECRET_ACCESS_KEY
+          production: false
           requires:
             - approve-push-unstable
           filters:
             branches:
               only: main
       - deploy-images:
+          name: Deploy images to unstable
+          postgres-password: DEV_POSTGRES_PASSWORD
+          mongodb-password: DEV_MONGO_INITDB_ROOT_PASSWORD
           requires:
-            - build-and-push
+            - build-and-push-unstable
   release:
     jobs:
       - linux-qa:
@@ -690,3 +749,125 @@ workflows:
           filters:
             branches:
               only: production
+      - approve-push-production:
+          type: approval
+          filters:
+            branches:
+              only: production
+      - build-and-push:
+          name: build-and-push-production
+          aws-access-key-id: PROD_AWS_ACCESS_KEY_ID
+          aws-secret-access-key: PROD_AWS_SECRET_ACCESS_KEY
+          production: true
+          requires:
+            - approve-push-production
+          filters:
+            branches:
+              only: production
+      - deploy-images:
+          name: deploy-images-to-production
+          postgres-password: PROD_POSTGRES_PASSWORD
+          mongodb-password: PROD_MONGO_INITDB_ROOT_PASSWORD
+          ssh-fingerprint: 6a:c5:33:fe:5b:c9:06:df:99:64:ca:17:0d:32:18:2e
+          ssh-config-script: production-ssh-config.sh
+          ssh-host: shuttle.prod.internal
+          production: true
+          requires:
+            - build-and-push-production
+          filters:
+            branches:
+              only: production
+      - approve-publish-crates:
+          type: approval
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          name: publish-shuttle-codegen
+          path: codegen
+          requires:
+            - approve-publish-crates
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          name: publish-shuttle-common
+          path: common
+          requires:
+            - publish-shuttle-codegen
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          name: publish-shuttle-proto
+          path: proto
+          requires:
+            - publish-shuttle-common
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          name: publish-shuttle-service
+          path: service
+          requires:
+            - publish-shuttle-proto
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          name: publish-shuttle-runtime
+          path: runtime
+          requires:
+            - publish-shuttle-service
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          matrix:
+            parameters:
+              path: 
+                [
+                  "resources/aws-rds",
+                  "resources/shared-db",
+                  "resources/secrets",
+                  "resources/persist",
+                  "resources/static-folder"
+                ]
+          name: publish-<< matrix.path >>
+          requires:
+            - publish-shuttle-runtime
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          name: publish-cargo-shuttle
+          path: cargo-shuttle
+          requires:
+            - publish-shuttle-service
+          filters:
+            branches:
+              only: production
+      - publish-crates:
+          matrix:
+            parameters:
+              path: 
+                [
+                  "services/shuttle-actix-web", 
+                  "services/shuttle-axum", 
+                  "services/shuttle-next", 
+                  "services/shuttle-poem", 
+                  "services/shuttle-poise",
+                  "services/shuttle-rocket",
+                  "services/shuttle-salvo",
+                  "services/shuttle-serenity",
+                  "services/shuttle-thruset",
+                  "services/shuttle-tide",
+                  "services/shuttle-tower",
+                  "services/shuttle-warp"
+                ]
+          name: publish-<< matrix.path >>
+          requires:
+            - publish-cargo-shuttle
+          filters:
+            branches:
+              only: production

.circleci/production-ssh-config.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env sh
+
+set -ue
+
+# add our config to the .ssh/config in circleci
+cat >> $HOME/.ssh/config <<- EOF
+Host admin
+    HostName 18.132.154.166
+    User ec2-user
+
+Host *.shuttle.prod.internal
+    User ec2-user
+    ProxyJump ec2-user@admin
+EOF