Upgrade Porter to v1 (#3014)

* Porter v1 core changes (#2977)
* Update templates for Porter v1 (#2985)
* Update docs for porter v1 (#2992)

.devcontainer/Dockerfile

@@ -11,9 +11,10 @@ ARG USERNAME=vscode
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 # Set up non-root user
 COPY .devcontainer/scripts/non-root-user.sh /tmp/
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN bash /tmp/non-root-user.sh "${USERNAME}" "${USER_UID}" "${USER_GID}"
 
 # Set env for tracking that we're running in a devcontainer
@@ -24,7 +25,7 @@ ARG NODE_VERSION="lts/*"
 RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # Install terraform
-ARG TERRAFORM_VERSION="1.3.5"
+ARG TERRAFORM_VERSION="1.3.6"
 COPY .devcontainer/scripts/terraform.sh /tmp/
 RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin
 
@@ -42,7 +43,7 @@ RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release
 
 # Install Certbot
 RUN if [ "${INTERACTIVE}" = "true" ]; then \
-    apt-get update && apt-get install -y python3 python3-venv libaugeas0 --no-install-recommends \
+    apt-get update && apt-get install -y libaugeas0 --no-install-recommends \
     && python3 -m venv /opt/certbot/ \
     && /opt/certbot/bin/pip install --no-cache-dir --upgrade pip \
     && /opt/certbot/bin/pip install --no-cache-dir certbot \
@@ -51,23 +52,39 @@ RUN if [ "${INTERACTIVE}" = "true" ]; then \
 # Install Porter
 # Not using the script from https://cdn.porter.sh/latest/install-linux.sh
 # as it installs things we don't need and duplicates the binary.
+# TODO: Remove Porter v0 https://github.com/microsoft/AzureTRE/issues/2990
 ARG PORTER_MIRROR=https://cdn.porter.sh
 ARG PORTER_VERSION=v0.38.13
 ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.0-rc.1
 ARG PORTER_AZ_MIXIN_VERSION=v0.7.3
 ARG PORTER_AZURE_PLUGIN_VERSION=v0.11.2
-ARG PORTER_HOME=/home/$USERNAME/.porter/
+ARG PORTER_HOME_V0=/home/$USERNAME/.porter-v0/
 COPY .devcontainer/scripts/porter.sh /tmp/
-RUN export PORTER_MIRROR=${PORTER_MIRROR} \
+RUN if [ "${INTERACTIVE}" = "true" ]; then \
+    export PORTER_MIRROR=${PORTER_MIRROR} \
     PORTER_VERSION=${PORTER_VERSION} \
     PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \
     PORTER_AZ_MIXIN_VERSION=${PORTER_AZ_MIXIN_VERSION} \
-    PORTER_DOCKER_MIXIN_VERSION=${PORTER_DOCKER_MIXIN_VERSION} \
     PORTER_AZURE_PLUGIN_VERSION=${PORTER_AZURE_PLUGIN_VERSION} \
-    PORTER_HOME=${PORTER_HOME} \
-    && /tmp/porter.sh
+    PORTER_HOME=${PORTER_HOME_V0} \
+    && /tmp/porter.sh ; fi
+
+# can't be in a non default path
+# ARG PORTER_HOME_V1=/home/$USERNAME/.porter-v1/
+ARG PORTER_HOME_V1=/home/$USERNAME/.porter/
+ARG PORTER_VERSION=v1.0.4
+ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.0
+ARG PORTER_AZ_MIXIN_VERSION=v1.0.0
+ARG PORTER_AZURE_PLUGIN_VERSION=v1.0.1
+COPY .devcontainer/scripts/porter-v1.sh /tmp/
+RUN export PORTER_VERSION=${PORTER_VERSION} \
+    PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \
+    PORTER_AZ_MIXIN_VERSION=${PORTER_AZ_MIXIN_VERSION} \
+    PORTER_AZURE_PLUGIN_VERSION=${PORTER_AZURE_PLUGIN_VERSION} \
+    PORTER_HOME=${PORTER_HOME_V1} \
+    && /tmp/porter-v1.sh
 
-ENV PATH ${PORTER_HOME}:$PATH
+ENV PATH ${PORTER_HOME_V1}:$PATH
 
 # Install requirements
 COPY ["requirements.txt", "/tmp/pip-tmp/" ]
@@ -76,15 +93,15 @@ COPY ["resource_processor/vmss_porter/requirements.txt", "/tmp/pip-tmp/resource_
 COPY ["docs/requirements.txt", "/tmp/pip-tmp/docs/"]
 COPY ["e2e_tests/requirements.txt", "/tmp/pip-tmp/e2e_tests/"]
 COPY ["airlock_processor/requirements.txt", "/tmp/pip-tmp/airlock_processor/"]
-RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt && rm -rf /tmp/pip-tmp
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
 
 # Install azure-cli
 ARG AZURE_CLI_VERSION=2.37.0-1~bullseye
 COPY .devcontainer/scripts/azure-cli.sh /tmp/
 RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \
     && /tmp/azure-cli.sh
 
-ARG YQ_VERSION="v4.27.2"
+ARG YQ_VERSION="v4.30.6"
 RUN curl -L --fail -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
     && chmod +x /usr/local/bin/yq
 

.devcontainer/devcontainer.json

@@ -21,7 +21,9 @@
     // Mounts the login details from the host machine to azcli works in the container
     "type=bind,source=${env:HOME}${env:USERPROFILE}/.azure,target=/home/vscode/.azure",
     // Mount docker socket for docker builds
-    "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock"
+    "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock",
+    // Mounts the github cli login details from the host machine to the container (~/.config/gh/hosts.yml)
+    "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config",
   ],
   "remoteUser": "vscode",
   "containerEnv": {

.devcontainer/scripts/porter-v1.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+# Uncomment this line to see each command for debugging (careful: this will show secrets!)
+# set -o xtrace
+
+export PORTER_HOME=${PORTER_HOME:-~/.porter}
+export PORTER_MIRROR=${PORTER_MIRROR:-https://cdn.porter.sh}
+PORTER_VERSION=${PORTER_VERSION:-latest}
+
+echo "Installing porter@$PORTER_VERSION to $PORTER_HOME from $PORTER_MIRROR"
+
+mkdir -p "$PORTER_HOME/runtimes"
+
+curl -fsSLo "$PORTER_HOME/porter" "$PORTER_MIRROR/$PORTER_VERSION/porter-linux-amd64"
+chmod +x "$PORTER_HOME/porter"
+ln -s "$PORTER_HOME/porter" "$PORTER_HOME/runtimes/porter-runtime"
+echo "Installed $("${PORTER_HOME}"/porter version)"
+
+"${PORTER_HOME}/porter" mixin install exec --version "$PORTER_VERSION"
+"${PORTER_HOME}/porter" mixin install terraform --version "$PORTER_TERRAFORM_MIXIN_VERSION"
+"${PORTER_HOME}/porter" mixin install az --version "$PORTER_AZ_MIXIN_VERSION"
+
+"${PORTER_HOME}/porter" plugin install azure --version "$PORTER_AZURE_PLUGIN_VERSION"
+
+chown -R "${USERNAME}" "${PORTER_HOME}"
+
+echo "Installation complete."

.dockerignore

@@ -1,5 +1,5 @@
 # See https://docs.docker.com/engine/reference/builder/#dockerignore-file
-# Put files here that you don't want copied into your bundle's invocation image
+# Put files here that you need in the devcontainer's context
 .gitignore
 Dockerfile.tmpl
 
@@ -18,8 +18,16 @@ cli/build
 cli/dist
 *.egg-info/
 
-.terraform
+**/.terraform
 tfplan*
 *.log
 
 templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure/target
+
+**/node_modules
+**/.cnab
+
+ui/app/build
+site
+
+.git

.github/workflows/build_docker_images.yml

@@ -3,7 +3,9 @@ name: Docker build
 
 on:  # yamllint disable-line rule:truthy
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - 'feature/**'
   workflow_dispatch:
 
 # for each ref (branch/pr) run just the most recent, cancel

.github/workflows/build_validation_develop.yml

@@ -3,7 +3,9 @@ name: Build Validation
 
 on:  # yamllint disable-line rule:truthy
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - 'feature/**'
 
 # for each ref (branch/pr) run just the most recent,
 # cancel other pending/running ones

.github/workflows/deploy_tre_reusable.yml

@@ -206,8 +206,12 @@ jobs:
         # failure in the first attempt indicates a new ACR, so we need to try again after it's been created
         if: steps.ci_cache_cr_login.outcome != 'success'
         run: |
-          # shellcheck disable=SC2034,SC2015
-          for i in {1..3}; do az acr login --name "${{ secrets.CI_CACHE_ACR_NAME }}" && break || sleep 10; done
+          # shellcheck disable=SC2034,SC2015,SC2125
+          for i in {1..3}; do
+            az acr login --name "${{ secrets.CI_CACHE_ACR_NAME }}" && ec=0 && break || ec=\$? && sleep 10
+          done
+          # shellcheck disable=SC2242
+          (exit \$ec)
 
       - name: Push cached devcontainer
         run: docker image push ${{ env.CI_CACHE_ACR_URI }}/tredev:${{ secrets.DEVCONTAINER_TAG }}
@@ -359,7 +363,7 @@ jobs:
           # Although porter publish will build automatically, our makefile build target includes logic that should run
           COMMAND: >-
             for i in {1..3}; do make bundle-build bundle-publish DIR=${{ matrix.BUNDLE_DIR }}
-            && break || sleep 30; done
+            && ec=0 && break || ec=\$? && sleep 30; done; (exit \$ec)
           DEVCONTAINER_TAG: ${{ secrets.DEVCONTAINER_TAG }}
           AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
           CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}}
@@ -392,7 +396,7 @@ jobs:
           # Although porter publish will build automatically, our makefile build target includes logic that should run
           COMMAND: >-
             for i in {1..3}; do make bundle-build bundle-publish DIR=${{ matrix.BUNDLE_DIR }}
-            && break || sleep 30; done
+            && ec=0 && break || ec=\$? && sleep 30; done; (exit \$ec)
           DEVCONTAINER_TAG: ${{ secrets.DEVCONTAINER_TAG }}
           AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
           CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}}

.gitignore

@@ -115,8 +115,9 @@ templates/core/tre.env
 devops/auth.env
 private.env
 
-# Config
+# TRE Config
 config.yaml
+!resource_processor/vmss_porter/config.yaml
 
 # Spyder project settings
 .spyderproject

CHANGELOG.md

@@ -4,9 +4,9 @@
 **BREAKING CHANGES & MIGRATIONS**:
 * The model for `reviewUserResources` in airlock requests has changed from being a list to a dictionary. A migration has been added to update your existing requests automatically; please make sure you run the migrations as part of updating your API and UI.
   * Note that any in-flight requests that have review resources deployed will show `UNKNOWN[i]` for the user key of that resource and in the UI users will be prompted to deploy a new resource. [#2883](https://github.com/microsoft/AzureTRE/pull/2883)
-* Env files consolidation - The files /templates/core/.env, /devops/.env, /devops/auth.env are no longer used. The settings and configuration that they contain has been consolidated into a single file config.yaml that lives in the root folder of the project.
-
+* Env files consolidation ([#2944](https://github.com/microsoft/AzureTRE/pull/2944)) - The files /templates/core/.env, /devops/.env, /devops/auth.env are no longer used. The settings and configuration that they contain has been consolidated into a single file config.yaml that lives in the root folder of the project.
 Use the script devops/scripts/env_to_yaml_config.sh to migrate /templates/core/.env, /devops/.env, and /devops/auth.env to the new config.yaml file.
+* Upgrade to Porter v1 ([#3014](https://github.com/microsoft/AzureTRE/pull/3014)). You should upgrade all custom template definitions and rebuild them.
 
 FEATURES:
 * Support review VMs for multiple reviewers for each airlock request [#2883](https://github.com/microsoft/AzureTRE/pull/2883)

Makefile

@@ -184,16 +184,22 @@ bundle-build:
 	&& if [ -d terraform ]; then terraform -chdir=terraform init -backend=false; terraform -chdir=terraform validate; fi \
 	&& FULL_IMAGE_NAME_PREFIX=${FULL_IMAGE_NAME_PREFIX} IMAGE_NAME_PREFIX=${IMAGE_NAME_PREFIX} \
 		${MAKEFILE_DIR}/devops/scripts/bundle_runtime_image_build.sh \
-	&& porter build --debug
+	&& porter build
 	$(MAKE) bundle-check-params
 
 bundle-install: bundle-check-params
 	$(call target_title, "Deploying ${DIR} with Porter") \
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh porter,env \
 	&& . ${MAKEFILE_DIR}/devops/scripts/load_and_validate_env.sh \
-	&& cd ${DIR} && porter install -p ./parameters.json \
-		--cred ${MAKEFILE_DIR}/resource_processor/vmss_porter/arm_auth_local_debugging.json \
-		--cred ${MAKEFILE_DIR}/resource_processor/vmss_porter/aad_auth_local_debugging.json \
+	&& cd ${DIR} \
+	&& . ${MAKEFILE_DIR}/devops/scripts/load_env.sh .env \
+	&& porter parameters apply parameters.json \
+	&& porter credentials apply ${MAKEFILE_DIR}/resource_processor/vmss_porter/aad_auth_local_debugging.json \
+	&& porter credentials apply ${MAKEFILE_DIR}/resource_processor/vmss_porter/arm_auth_local_debugging.json \
+	&& . ${MAKEFILE_DIR}/devops/scripts/porter_local_env.sh \
+	&& porter install --parameter-set $$(yq ".name" porter.yaml) \
+		--credential-set arm_auth \
+		--credential-set aad_auth \
 		--allow-docker-host-access --debug
 
 # Validates that the parameters file is synced with the bundle.
@@ -204,27 +210,40 @@ bundle-check-params:
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh nodocker,porter \
 	&& cd ${DIR} \
 	&& if [ ! -f "parameters.json" ]; then echo "Error - please create a parameters.json file."; exit 1; fi \
-	&& if ! porter explain -ojson > /dev/null; then echo "Error - porter explain issue!"; exit 1; fi \
+	&& if [ "$$(jq -r '.name' parameters.json)" != "$$(yq eval '.name' porter.yaml)" ]; then echo "Error - ParameterSet name isn't equal to bundle's name."; exit 1; fi \
+	&& if ! porter explain; then echo "Error - porter explain issue!"; exit 1; fi \
 	&& comm_output=$$(set -o pipefail && comm -3 --output-delimiter=: <(porter explain -ojson | jq -r '.parameters[].name | select (. != "arm_use_msi")' | sort) <(jq -r '.parameters[].name | select(. != "arm_use_msi")' parameters.json | sort)) \
 	&& if [ ! -z "$${comm_output}" ]; \
-		then echo -e "*** Add to params ***:*** Remove from params ***\n$$comm_output" | column -t -s ":" -n; exit 1; \
+		then echo -e "*** Add to params ***:*** Remove from params ***\n$$comm_output" | column -t -s ":"; exit 1; \
 		else echo "parameters.json file up-to-date."; fi
 
 bundle-uninstall:
 	$(call target_title, "Uninstalling ${DIR} with Porter") \
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh porter,env \
-	&& cd ${DIR} && porter uninstall -p ./parameters.json \
-		--cred ${MAKEFILE_DIR}/resource_processor/vmss_porter/arm_auth_local_debugging.json \
-		--cred ${MAKEFILE_DIR}/resource_processor/vmss_porter/aad_auth_local_debugging.json \
+	&& . ${MAKEFILE_DIR}/devops/scripts/load_and_validate_env.sh \
+	&& cd ${DIR} \
+	&& . ${MAKEFILE_DIR}/devops/scripts/load_env.sh .env \
+	&& porter parameters apply parameters.json \
+	&& porter credentials apply ${MAKEFILE_DIR}/resource_processor/vmss_porter/aad_auth_local_debugging.json \
+	&& porter credentials apply ${MAKEFILE_DIR}/resource_processor/vmss_porter/arm_auth_local_debugging.json \
+	&& porter uninstall --parameter-set $$(yq ".name" porter.yaml) \
+		--credential-set arm_auth \
+		--credential-set aad_auth \
 		--allow-docker-host-access --debug
 
 bundle-custom-action:
  	$(call target_title, "Performing:${ACTION} ${DIR} with Porter") \
  	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh porter,env \
- 	&& cd ${DIR} && porter invoke --action ${ACTION} -p ./parameters.json \
- 		--cred ${MAKEFILE_DIR}/resource_processor/vmss_porter/arm_auth_local_debugging.json \
- 		--cred ${MAKEFILE_DIR}/resource_processor/vmss_porter/aad_auth_local_debugging.json \
- 		--allow-docker-host-access --debug
+	&& . ${MAKEFILE_DIR}/devops/scripts/load_and_validate_env.sh \
+	&& cd ${DIR}
+	&& . ${MAKEFILE_DIR}/devops/scripts/load_env.sh .env \
+	&& porter parameters apply parameters.json \
+	&& porter credentials apply ${MAKEFILE_DIR}/resource_processor/vmss_porter/aad_auth_local_debugging.json \
+	&& porter credentials apply ${MAKEFILE_DIR}/resource_processor/vmss_porter/arm_auth_local_debugging.json \
+ 	&& porter invoke --action ${ACTION} --parameter-set $$(yq ".name" porter.yaml) \
+		--credential-set arm_auth \
+		--credential-set aad_auth \
+		--allow-docker-host-access --debug
 
 bundle-publish:
 	$(call target_title, "Publishing ${DIR} bundle with Porter") \
@@ -234,7 +253,7 @@ bundle-publish:
 	&& cd ${DIR} \
 	&& FULL_IMAGE_NAME_PREFIX=${FULL_IMAGE_NAME_PREFIX} \
 		${MAKEFILE_DIR}/devops/scripts/bundle_runtime_image_push.sh \
-	&& porter publish --registry "$${ACR_NAME}.azurecr.io" --debug
+	&& porter publish --registry "$${ACR_NAME}.azurecr.io" --force
 
 bundle-register:
 	@# NOTE: ACR_NAME below comes from the env files, so needs the double '$$'. Others are set on command execution and don't
@@ -290,11 +309,11 @@ build-and-deploy-ui:
 	&& if [ "$${DEPLOY_UI}" != "false" ]; then ${MAKEFILE_DIR}/devops/scripts/build_deploy_ui.sh; else echo "UI Deploy skipped as DEPLOY_UI is false"; fi \
 
 prepare-for-e2e:
-	$(MAKE) workspace_bundle BUNDLE=base \
-	&& $(MAKE) workspace_service_bundle BUNDLE=guacamole \
-	&& $(MAKE) shared_service_bundle BUNDLE=gitea \
-	&& $(MAKE) user_resource_bundle WORKSPACE_SERVICE=guacamole BUNDLE=guacamole-azure-windowsvm \
-	&& $(MAKE) user_resource_bundle WORKSPACE_SERVICE=guacamole BUNDLE=guacamole-azure-linuxvm
+	$(MAKE) workspace_bundle BUNDLE=base
+	$(MAKE) workspace_service_bundle BUNDLE=guacamole
+	$(MAKE) shared_service_bundle BUNDLE=gitea
+	$(MAKE) user_resource_bundle WORKSPACE_SERVICE=guacamole BUNDLE=guacamole-azure-windowsvm
+	$(MAKE) user_resource_bundle WORKSPACE_SERVICE=guacamole BUNDLE=guacamole-azure-linuxvm
 
 test-e2e-smoke:
 	$(call target_title, "Running E2E smoke tests") && \

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.7.0"
+__version__ = "0.8.0"

api_app/models/domain/resource.py

@@ -75,5 +75,5 @@ def parse_etag_to_remove_escaped_quotes(cls, value):
 
 
 class Output(AzureTREModel):
-    Name: str = Field(title="", description="")
-    Value: Union[list, dict, str] = Field(None, title="", description="")
+    Name: str = Field(title="", description="", alias="name")
+    Value: Union[list, dict, str] = Field(None, title="", description="", alias="value")

api_app/service_bus/deployment_status_updater.py

@@ -81,10 +81,11 @@ async def process_message(self, msg):
             complete_message = await self.update_status_in_database(message)
             logging.info(f"Update status in DB for {message.operationId} - {message.status}")
         except (json.JSONDecodeError, ValidationError) as e:
+            # TODO: should move to dead letter queue https://github.com/microsoft/AzureTRE/issues/2991
             complete_message = True
             logging.error(f"{strings.DEPLOYMENT_STATUS_MESSAGE_FORMAT_INCORRECT}: {msg.correlation_id} - {e}")
-        except Exception as e:
-            logging.info(f"Exception for: {msg.correlation_id} - {e}")
+        except Exception:
+            logging.exception(f"Exception processing message: {msg.correlation_id}")
 
         return complete_message
 

devops/scripts/porter_local_env.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# This script adds missing env vars that are needed to run porter commands locally.
+# If a bundle defines a parameter that isn't in the environment it will be added.
+# When/if this issue will be address, we could remove the script:
+# https://github.com/getporter/porter/issues/2474
+
+set -o errexit
+set -o pipefail
+# set -o xtrace
+
+while read -r env_var_name; do
+  if [[ -z "${!env_var_name}" ]]; then
+    echo "${env_var_name} doesn't exist."
+    # shellcheck disable=SC2086
+    declare -g $env_var_name=
+    export "${env_var_name?}"
+  fi
+done < <(jq -r '.parameters[].source.env' parameters.json)