Bicep compiler merge (#7662)

# Description This PR contains all the changes for the Bicep compiler merge. The branch has had smaller PRs opened for specific features that can be found below: - [x] Bicep types generator is updated to use the `bicep-types` repository as a submodule. This includes updating the types contract and adding support for resource methods (eg `listSecrets`): 43403fd . There was also a small update to how we handle flags on the `location` [property](19afe96) - [x] Update to build workflow to initialize the submodule and publish types to `biceptypes.azurecr.io` using the `bicep publish-extension` command: fed2462 - [x] Update validate bicep workflow to validate bicep files in the repository with the official Bicep compiler. This includes updating extension imports of all bicep files and adding a `bicepconfig.json` to the repository: #7702 - [x] Update functional test workflow to publish types to `biceptypes.azurecr.io`. This includes publishing the bicep types to a PR tag and adding a temporary `bicepconfig.json` to consume the PR-specific types [#7730] - [x] Update release workflow to publish types to a release-specific tag. This includes running a build in the `bicep-types-aws` repo to publish AWS types to a release tag - [x] Updates to `rad bicep download`. This command will download the official Bicep binary instead of the Radius one. The install script calls this command so the official Bicep binary will be downloaded on a Radius install: bb8da88 - [x] Updates to `rad init` to generate a default `bicepconfig.json`. This will pull the Radius and AWS types in our ACR with the current release tag: #7664 ## Type of change  - This pull request fixes a bug in Radius and has an approved issue (issue link required). - This pull request adds or changes features of Radius and has an approved issue (issue link required). - This pull request is a minor refactor, code cleanup, test improvement, or other maintenance task and doesn't change the functionality of Radius (issue link optional).  Fixes: #issue_number --------- # Description _Please explain the changes you've made._ ## Type of change  - This pull request fixes a bug in Radius and has an approved issue (issue link required). - This pull request adds or changes features of Radius and has an approved issue (issue link required). - This pull request is a minor refactor, code cleanup, test improvement, or other maintenance task and doesn't change the functionality of Radius (issue link optional).  Fixes: #issue_number --------- Signed-off-by: sk593 <shruthikumar@microsoft.com>
radius-project · Aug 12, 2024 · 5c10d92 · 5c10d92
1 parent 3d96ee6
commit 5c10d92
Show file tree

Hide file tree

Showing 156 changed files with 12,392 additions and 4,440 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -65,6 +65,9 @@ env:
   # URL to get source code for building the image
   IMAGE_SRC: https://github.com/radius-project/radius
 
+  # bicep-types ACR url for uploading Radius Bicep types
+  BICEP_TYPES_REGISTRY: 'biceptypes.azurecr.io'
+
 jobs:
   build-and-push-cli:
     name: Build ${{ matrix.target_os }}_${{ matrix.target_arch }} binaries
@@ -259,6 +262,50 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | helm registry login -u ${{ github.actor }} --password-stdin ${{ env.OCI_REGISTRY }}
           helm push ${{ env.ARTIFACT_DIR }}/${{ env.HELM_PACKAGE_DIR }}/radius-${{ env.CHART_VERSION }}.tgz oci://${{ env.OCI_REGISTRY }}/${{ env.OCI_REPOSITORY }}
 
+  build-and-push-bicep-types:
+    name: Publish Radius bicep types to ACR
+    runs-on: ubuntu-latest
+    steps: 
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Parse release version and set environment variables
+        run: python ./.github/scripts/get_release_version.py
+      - name: Set up Go ${{ env.GOVER }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOVER }}
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+      - name: Generate Bicep extensibility types from OpenAPI specs
+        run: |
+          make generate-bicep-types VERSION=${{ env.REL_CHANNEL == 'edge' && 'latest' || env.REL_CHANNEL }}
+      - name: Upload Radius Bicep types artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: radius-bicep-types
+          path: ./hack/bicep-types-radius/generated
+          if-no-files-found: error
+      - name: 'Login via Azure CLI'
+        if: github.repository == 'radius-project/radius' && ((startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'))
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.BICEPTYPES_CLIENT_ID }}
+          tenant-id: ${{ secrets.BICEPTYPES_TENANT_ID }}
+          subscription-id: ${{ secrets.BICEPTYPES_SUBSCRIPTION_ID }}
+      - name: Setup and verify bicep CLI
+        if: github.repository == 'radius-project/radius' && ((startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'))
+        run: |
+          curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64
+          chmod +x ./bicep
+          sudo mv ./bicep /usr/local/bin/bicep
+          bicep --version
+      - name: Publish bicep types 
+        if: github.repository == 'radius-project/radius' && ((startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'))
+        run: |
+          bicep publish-extension ./hack/bicep-types-radius/generated/index.json --target br:${{ env.BICEP_TYPES_REGISTRY }}/radius:${{ env.REL_CHANNEL == 'edge' && 'latest' || env.REL_CHANNEL }} --force
+
   publish-release:
     name: Publish GitHub Release
     needs: ["build-and-push-cli"]

diff --git a/.github/workflows/functional-test-cloud.yaml b/.github/workflows/functional-test-cloud.yaml
@@ -90,6 +90,8 @@ env:
   TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"
   # The number of failed tests to report.
   ISSUE_CREATE_THRESHOLD: 2
+  # bicep-types ACR url for uploading Radius Bicep types
+  BICEP_TYPES_REGISTRY: 'biceptypes.azurecr.io'
 
 jobs:
   build:
@@ -348,10 +350,51 @@ jobs:
           append: true
           message: |
             :x: Test recipe publishing failed
+  
+  publish-test-bicep-types:
+    name: Publish Radius bicep types to ACR
+    runs-on: ubuntu-latest
+    steps: 
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Parse release version and set environment variables
+        run: python ./.github/scripts/get_release_version.py
+      - name: Set up Go ${{ env.GOVER }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOVER }}
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+      - name: Generate Bicep extensibility types from OpenAPI specs
+        run: |
+          make generate-bicep-types VERSION=${{ env.REL_VERSION == 'edge' && 'latest' || env.REL_VERSION }}
+      - name: Upload Radius Bicep types artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.name }}_radius_bicep_types
+          path: ./hack/bicep-types-radius/generated
+          if-no-files-found: error
+      - name: 'Login via Azure CLI'
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.BICEPTYPES_CLIENT_ID }}
+          tenant-id: ${{ secrets.BICEPTYPES_TENANT_ID }}
+          subscription-id: ${{ secrets.BICEPTYPES_SUBSCRIPTION_ID }}
+      - name: Setup and verify bicep CLI
+        run: |
+          curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64
+          chmod +x ./bicep
+          sudo mv ./bicep /usr/local/bin/bicep
+          bicep --version
+      - name: Publish bicep types 
+        run: |
+          bicep publish-extension ./hack/bicep-types-radius/generated/index.json --target br:${{ env.BICEP_TYPES_REGISTRY }}/test/radius:${{ env.REL_VERSION == 'edge' && 'latest' || env.REL_VERSION }} --force
 
   tests:
     name: Run ${{ matrix.name }} functional tests
-    needs: build
+    needs: [build, publish-test-bicep-types]
     if: github.event_name == 'repository_dispatch' || (github.event_name == 'schedule' && github.repository == 'radius-project/radius') || github.event_name == 'workflow_run'
     strategy:
       fail-fast: true
@@ -593,7 +636,29 @@ jobs:
       - name: Publish Terraform test recipes
         run: |
           make publish-test-terraform-recipes
-
+      - name: Generate test bicepconfig.json
+        run: |
+          if [[ "${{ env.REL_VERSION }}" == "edge" ]]; then
+            RADIUS_VERSION="latest"
+          else
+            RADIUS_VERSION="${{ env.REL_VERSION }}"
+          fi
+          cat <<EOF > ./test/bicepconfig.json
+          {
+            "experimentalFeaturesEnabled": {
+              "extensibility": true,
+              "extensionRegistry": true,
+              "dynamicTypeLoading": true
+            },
+            "extensions": {
+              "radius": "br:${{ env.BICEP_TYPES_REGISTRY }}/test/radius:$RADIUS_VERSION",
+              "aws": "br:${{ env.BICEP_TYPES_REGISTRY }}/aws:latest"
+            },
+            "cloud": {
+              "credentialPrecedence": ["Environment"]
+            }
+          }
+          EOF
       - name: Run functional tests
         run: |
           # Ensure rad cli is in path before running tests.

diff --git a/.github/workflows/functional-test-noncloud.yaml b/.github/workflows/functional-test-noncloud.yaml
@@ -80,6 +80,8 @@ env:
   LOCAL_REGISTRY_SERVER: "localhost"
   # Local Docker registry port
   LOCAL_REGISTRY_PORT: "5000"
+  # bicep-types ACR url for uploading Radius Bicep types
+  BICEP_TYPES_REGISTRY: 'biceptypes.azurecr.io'
 
 jobs:
   build:
@@ -125,8 +127,7 @@ jobs:
           # Set output variables to be used in the other jobs
           echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_OUTPUT
           echo "DE_IMAGE=${{ env.DE_IMAGE }}" >> $GITHUB_OUTPUT
-          echo "DE_TAG=${{ env.DE_TAG }}" >> $GITHUB_OUTPUT
-
+          echo "DE_TAG=${{ env.DE_TAG }}" >> $GITHUB_OUTPUT    
   tests:
     name: Run ${{ matrix.name }} functional tests
     needs: build
@@ -205,6 +206,22 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+
+      - name: Generate Bicep extensibility types from OpenAPI specs
+        run: |
+          make generate-bicep-types VERSION=${{ env.REL_VERSION == 'edge' && 'latest' || env.REL_VERSION }}
+
+      - name: Upload Radius Bicep types artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.name }}_radius_bicep_types
+          path: ./hack/bicep-types-radius/generated
+          if-no-files-found: error       
+
       - name: Create a secure local registry
         id: create-local-registry
         uses: ./.github/actions/create-local-registry
@@ -300,6 +317,41 @@ jobs:
         run: |
           make publish-test-terraform-recipes
 
+      - name: Setup and verify bicep CLI
+        run: |
+          curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64
+          chmod +x ./bicep
+          sudo mv ./bicep /usr/local/bin/bicep
+          bicep --version
+
+      - name: Publish bicep types 
+        run: | 
+          bicep publish-extension ./hack/bicep-types-radius/generated/index.json --target br:${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}/radius:${{ env.REL_VERSION == 'edge' && 'latest' || env.REL_VERSION }} --force     
+
+      - name: Generate test bicepconfig.json
+        run: |
+          if [[ "${{ env.REL_VERSION }}" == "edge" ]]; then
+            RADIUS_VERSION="latest"
+          else
+            RADIUS_VERSION="${{ env.REL_VERSION }}"
+          fi
+          cat <<EOF > ./test/bicepconfig.json
+          {
+            "experimentalFeaturesEnabled": {
+              "extensibility": true,
+              "extensionRegistry": true,
+              "dynamicTypeLoading": true
+            },
+            "extensions": {
+              "radius": "br:${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}/radius:$RADIUS_VERSION",
+              "aws": "br:${{ env.BICEP_TYPES_REGISTRY }}/aws:latest"
+            },
+            "cloud": {
+              "credentialPrecedence": ["Environment"]
+            }
+          }
+          EOF
+
       - name: Publish Bicep Test Recipes
         run: |
           export PATH=$GITHUB_WORKSPACE/bin:$PATH
@@ -310,7 +362,7 @@ jobs:
           BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
           TEMP_CERT_DIR: ${{ steps.create-local-registry.outputs.temp-cert-dir }}
           SSL_CERT_FILE: ${{ steps.create-local-registry.outputs.temp-cert-dir }}/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt
-
+      
       - name: Run functional tests
         run: |
           # Ensure rad cli is in path before running tests.

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -159,6 +159,13 @@ jobs:
           ref: main
           token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           path: dashboard
+      - name: Checkout radius-project/bicep-types-aws@main
+        uses: actions/checkout@v4
+        with:
+          repository: radius-project/bicep-types-aws
+          ref: main
+          token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
+          path: bicep-types-aws
       - name: Set up GitHub credentials
         run: |
           git config --global user.name "Radius CI Bot"
@@ -226,3 +233,7 @@ jobs:
         if: success() && steps.release-branch-exists.outputs.result == 'false'
         run: |
           ./radius/.github/scripts/release-create-tag-and-branch.sh dashboard ${{ steps.get-version.outputs.release-version }} ${{ steps.get-version.outputs.release-branch-name }}
+      - name: Release radius-project/bicep-types-aws version ${{ steps.get-version.outputs.release-version }}
+        if: success() && steps.release-branch-exists.outputs.result == 'false'
+        run: |
+          ./radius/.github/scripts/release-create-tag-and-branch.sh bicep-types-aws ${{ steps.get-version.outputs.release-version }} ${{ steps.get-version.outputs.release-branch-name }}
diff --git a/.github/workflows/validate-bicep.yaml b/.github/workflows/validate-bicep.yaml
@@ -23,6 +23,9 @@ on:
       - main
       - release/*
 
+permissions:
+  id-token: write # Required for requesting the JWT
+
 concurrency:
   # Cancel the previously triggered build for only PR build.
   group: bicep-${{ github.event.pull_request.number || github.sha }}
@@ -33,17 +36,15 @@ jobs:
     name: Validate Bicep Code
     runs-on: ubuntu-latest
     steps:
+      - name: Setup and verify bicep CLI
+        run: |
+          curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64
+          chmod +x ./bicep
+          sudo mv ./bicep /usr/local/bin/bicep
+          bicep --version
       - name: Check out repo
         uses: actions/checkout@v4
-      - name: Parse release version and set environment variables
-        run: python ./.github/scripts/get_release_version.py
-      - uses: oras-project/setup-oras@main
-      - name: Download rad-bicep
-        run: |
-          oras pull ghcr.io/radius-project/radius/bicep/rad-bicep/linux-x64:latest -o ./
-          chmod +x rad-bicep
-          ./rad-bicep --version
       - name: Verify Bicep files
         run: ./build/validate-bicep.sh
         env:
-          BICEP_PATH: './rad-bicep'
+          BICEP_PATH: 'bicep'
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,7 @@ vendor/
 # Radius scaffolding
 app.bicep
 .rad/
+bicepconfig.json
 
 # Build Output
 bin/

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "bicep-types"]
+	path = bicep-types
+	url = https://github.com/Azure/bicep-types
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -87,6 +87,21 @@
                 "RADIUSBACKENDURI": "http://localhost:9000",
             },
         },
+        {
+            "name": "Debug Bicep generator integration tests",
+            "type": "node",
+            "request": "launch",
+            "runtimeArgs": [
+              "--inspect-brk",
+              "${workspaceRoot}/hack/bicep-types-radius/src/autorest.bicep/node_modules/.bin/jest",
+              "--runInBand",
+              "--no-cache"
+            ],
+            "cwd": "${workspaceFolder}/hack/bicep-types-radius/src/autorest.bicep/src",
+            "console": "integratedTerminal",
+            "internalConsoleOptions": "neverOpen",
+            "sourceMaps": true
+        },
     ],
     "compounds": [
         {

diff --git a/README.md b/README.md
@@ -63,7 +63,6 @@ In addition, we have the below repositories.
 | [Samples](https://github.com/radius-project/samples) | This repository contains the source code for quickstarts, reference apps, and tutorials for Radius.
 | [Recipes](https://github.com/radius-project/recipes) | This repo contains commonly used Recipe templates for Radius Environments.
 | [Website](https://github.com/radius-project/website) | This repository contains the source code for the Radius website.
-| [Bicep](https://github.com/radius-project/bicep) | This repository contains source code for Bicep, which is a DSL for deploying cloud resources types. 
 | [AWS Bicep Types](https://github.com/radius-project/bicep-types-aws) | This repository contains the tooling for Bicep support for AWS resource types.
 
 

diff --git a/bicep-types b/bicep-types
diff --git a/bicepconfig.json b/bicepconfig.json
@@ -0,0 +1,14 @@
+{
+	"experimentalFeaturesEnabled": {
+		"extensibility": true,
+		"extensionRegistry": true,
+		"dynamicTypeLoading": true
+	},
+	"extensions": {
+		"radius": "br:biceptypes.azurecr.io/radius:latest",
+		"aws": "br:biceptypes.azurecr.io/aws:latest"
+	},
+	"cloud": {
+		"credentialPrecedence": ["Environment"]
+	}
+}
diff --git a/build/generate.mk b/build/generate.mk
@@ -116,11 +116,12 @@ generate-go: generate-mockgen-installed ## Generates go with 'go generate' (Mock
 generate-bicep-types: generate-node-installed ## Generate Bicep extensibility types
 	@echo "$(ARROW) Generating Bicep extensibility types from OpenAPI specs..."
 	@echo "$(ARROW) Build autorest.bicep..."
-	cd hack/bicep-types-radius/src/autorest.bicep; \
-	npm ci && npm run build; \
-	cd ../generator; \
+	git submodule update --init --recursive; \
+	npm --prefix bicep-types/src/bicep-types install; \
+	npm --prefix bicep-types/src/bicep-types ci && npm --prefix bicep-types/src/bicep-types run build; \
+	npm --prefix hack/bicep-types-radius/src/autorest.bicep ci && npm --prefix hack/bicep-types-radius/src/autorest.bicep run build; \
 	echo "Run generator from hack/bicep-types-radius/src/generator dir"; \
-	npm ci && npm run generate -- --specs-dir ../../../../swagger --verbose
+	npm --prefix hack/bicep-types-radius/src/generator ci && npm --prefix hack/bicep-types-radius/src/generator run generate -- --specs-dir ../../../../swagger --release-version ${VERSION} --verbose
 
 # go-get-tool will 'go get' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))/..