Upmerge 0.33 to edge (#1563)

* update gh to ado sync workflow to use service principals (#1363) * Make env configurable (#1046) Signed-off-by: Reshma Abdul Rahim <reshmarahim.abdul@microsoft.com> * Adding test AKS workflow to v0.32 (#1052) * Run functional tests every 2 hours and add another workflow that runs… (#1020) * Run functional tests every 2 hours and add another workflow that runs the tests on AKS instead of k3d Signed-off-by: ytimocin <ytimocin@microsoft.com> * Triggering workflow Signed-off-by: ytimocin <ytimocin@microsoft.com> --------- Signed-off-by: ytimocin <ytimocin@microsoft.com> * Removing the run of the AKS workflow on PRs (#1051) Signed-off-by: ytimocin <ytimocin@microsoft.com> * Update the timeout (30s) for Playwright Signed-off-by: ytimocin <ytimocin@microsoft.com> --------- Signed-off-by: ytimocin <ytimocin@microsoft.com> * Use retry and update eshop playwright tests to wait for the catalog to appear (#1213) Signed-off-by: ytimocin <ytimocin@microsoft.com> * update gh to ado sync workflow to use service principals Signed-off-by: Will Tsai <28876888+willtsai@users.noreply.github.com> --------- Signed-off-by: Reshma Abdul Rahim <reshmarahim.abdul@microsoft.com> Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: Will Tsai <28876888+willtsai@users.noreply.github.com> Signed-off-by: Will <28876888+willtsai@users.noreply.github.com> Co-authored-by: Reshma Abdul Rahim <61033581+Reshrahim@users.noreply.github.com> Co-authored-by: Yetkin Timocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Workflow and playwright test updates (#1360) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Changing the place of page.reload in the tests (#1492) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Adding a job that will purge AWS EKS clusters every 6 hours (#1462) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Update all deps of all apps (#1536) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Updating the runner of purge azure test resources workflow (#1537) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Update purget test resources to fix the workflow failures (#1552) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> * Fixing the eShop catalog load issue for the UI tests (#1548) Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> --------- Signed-off-by: Reshma Abdul Rahim <reshmarahim.abdul@microsoft.com> Signed-off-by: ytimocin <ytimocin@microsoft.com> Signed-off-by: Will Tsai <28876888+willtsai@users.noreply.github.com> Signed-off-by: Will <28876888+willtsai@users.noreply.github.com> Signed-off-by: sk593 <shruthikumar@microsoft.com> Co-authored-by: Will <28876888+willtsai@users.noreply.github.com> Co-authored-by: Reshma Abdul Rahim <61033581+Reshrahim@users.noreply.github.com> Co-authored-by: Yetkin Timocin <ytimocin@microsoft.com>
radius-project · May 23, 2024 · b9018d4 · b9018d4
1 parent 6949da2
commit b9018d4
Show file tree

Hide file tree

Showing 32 changed files with 21,566 additions and 15,119 deletions.
diff --git a/.github/scripts/delete-aws-resources.sh b/.github/scripts/delete-aws-resources.sh
@@ -6,7 +6,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#    
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ -16,7 +16,6 @@
 # limitations under the License.
 # ------------------------------------------------------------
 
-
 APP_ID=$1
 APP_LABEL='radapp.io/application'
 RESOURCE_TYPES='AWS::RDS::DBInstance,AWS::RDS::DBSubnetGroup,AWS::MemoryDB::Cluster,AWS::MemoryDB::SubnetGroup'
@@ -34,21 +33,16 @@ function delete_aws_resources() {
   # Empty the file
   truncate -s 0 $DELETED_RESOURCES_FILE
 
-  for resource_type in ${RESOURCE_TYPES//,/ }
-  do
-    aws cloudcontrol list-resources --type-name "$resource_type" --query "ResourceDescriptions[].Identifier" --output text | tr '\t' '\n' | while read identifier
-    do
-      aws cloudcontrol get-resource --type-name "$resource_type" --identifier "$identifier" --query "ResourceDescription.Properties" --output text | while read resource
-      do
-        resource_tags=$(jq -c -r .Tags <<< "$resource")
-        for tag in $(jq -c -r '.[]' <<< "$resource_tags")
-        do
-          key=$(jq -r '.Key' <<< "$tag")
-          value=$(jq -r '.Value' <<< "$tag")
-          if [[ "$key" == "$APP_LABEL" && "$value" == "$APP_ID" ]]
-          then
+  for resource_type in ${RESOURCE_TYPES//,/ }; do
+    aws cloudcontrol list-resources --type-name "$resource_type" --query "ResourceDescriptions[].Identifier" --output text | tr '\t' '\n' | while read identifier; do
+      aws cloudcontrol get-resource --type-name "$resource_type" --identifier "$identifier" --query "ResourceDescription.Properties" --output text | while read resource; do
+        resource_tags=$(jq -c -r .Tags <<<"$resource")
+        for tag in $(jq -c -r '.[]' <<<"$resource_tags"); do
+          key=$(jq -r '.Key' <<<"$tag")
+          value=$(jq -r '.Value' <<<"$tag")
+          if [[ "$key" == "$APP_LABEL" && "$value" == "$APP_ID" ]]; then
             echo "Deleting resource of type: $resource_type with identifier: $identifier"
-            echo "$identifier\n" >> $DELETED_RESOURCES_FILE
+            echo "$identifier\n" >>$DELETED_RESOURCES_FILE
             aws cloudcontrol delete-resource --type-name "$resource_type" --identifier "$identifier"
           fi
         done
@@ -65,28 +59,28 @@ function delete_aws_resources() {
 
 RETRY_COUNT=0
 while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
-    # Trigger the function to delete the resources
-    delete_aws_resources
+  # Trigger the function to delete the resources
+  delete_aws_resources
 
-    # If the function returned 0, then no resources needed to be deleted
-    # on this run. This means that all resources have been deleted.
-    if [ $? -eq 0 ]; then
-        echo "All resources deleted successfully"
-        break
-    fi
+  # If the function returned 0, then no resources needed to be deleted
+  # on this run. This means that all resources have been deleted.
+  if [ $? -eq 0 ]; then
+    echo "All resources deleted successfully"
+    break
+  fi
 
-    # Still have resources to delete, increase the retry count
-    RETRY_COUNT=$((RETRY_COUNT + 1))
+  # Still have resources to delete, increase the retry count
+  RETRY_COUNT=$((RETRY_COUNT + 1))
 
-    # Check if there are more retries left
-    if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
-        # Retry after delay
-        echo "Retrying in $RETRY_DELAY seconds..."
-        sleep $RETRY_DELAY
-    fi
+  # Check if there are more retries left
+  if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
+    # Retry after delay
+    echo "Retrying in $RETRY_DELAY seconds..."
+    sleep $RETRY_DELAY
+  fi
 done
 
 # Check if the maximum number of retries exceeded
 if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
-    echo "Maximum number of retries exceeded"
+  echo "Maximum number of retries exceeded"
 fi
diff --git a/.github/scripts/purge-aws-eks-clusters.sh b/.github/scripts/purge-aws-eks-clusters.sh
@@ -0,0 +1,47 @@
+# ------------------------------------------------------------
+# Copyright 2023 The Radius Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ------------------------------------------------------------
+
+#!/bin/bash
+
+# Current time in seconds since epoch
+current_time=$(date +%s)
+
+# Age limit in seconds (6 hours * 3600 seconds/hour)
+age_limit=$((6 * 3600))
+
+echo "Starting cluster purge script."
+
+# List clusters and their creation times, filter and delete those older than 6 hours and starting with 'eks-samplestest-'
+aws eks list-clusters --query "clusters[]" --output text | xargs -I {} aws eks describe-cluster --name {} --query "cluster.{name: name, createdAt: createdAt}" --output text | while read -r created_at name; do
+    # Convert creation time to seconds since the epoch
+    # Remove milliseconds and adjust timezone format from "-07:00" to "-0700"
+    formatted_created_at="${created_at%.*}${created_at##*.}"
+    formatted_created_at="${formatted_created_at%:*}${formatted_created_at##*:}"
+
+    # Convert creation time to seconds
+    created_at_seconds=$(date -d "$formatted_created_at" +%s)
+
+    # Calculate age in seconds
+    age=$((current_time - created_at_seconds))
+
+    # Check if age is greater than age limit and name starts with 'eks-samplestest-'
+    if [ "$age" -gt "$age_limit" ] && [[ "$name" == eks-samplestest-* ]]; then
+        echo "Deleting cluster $name older than 6 hours."
+        eksctl delete cluster --name "$name" --wait --force
+    else
+        echo "Cluster $name is not older than 6 hours or does not meet naming criteria."
+    fi
+done
diff --git a/.github/scripts/purge-aws-rds-snapshots.sh b/.github/scripts/purge-aws-rds-snapshots.sh
@@ -4,7 +4,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#    
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ -16,8 +16,7 @@
 
 set -xe
 
-aws rds describe-db-snapshots --query 'DBSnapshots[].DBSnapshotIdentifier' --output text > snapshots.txt
-for rds_snapshot_identifier in $(cat ./snapshots.txt)    
-do
+aws rds describe-db-snapshots --query 'DBSnapshots[].DBSnapshotIdentifier' --output text >snapshots.txt
+for rds_snapshot_identifier in $(cat ./snapshots.txt); do
     aws rds delete-db-snapshot --db-snapshot-identifier $rds_snapshot_identifier
 done
diff --git a/.github/workflows/issues.yaml b/.github/workflows/issues.yaml
@@ -1,28 +1,49 @@
-name: Issues Automation
+name: Sync issue to Azure DevOps work item
 
 on:
   issues:
-    types: [opened, edited, deleted, closed, reopened, labeled, unlabeled, assigned]
-
+    types:
+      [opened, edited, deleted, closed, reopened, labeled, unlabeled, assigned]
+
 concurrency:
   group: issue-${{ github.event.issue.number }}
   cancel-in-progress: false
 
+# Extra permissions needed to login with Entra ID service principal via federated identity
+permissions:
+  id-token: write
+  issues: write
+
 jobs:
   ado:
-    name: Sync issue to Azure DevOps
     runs-on: ubuntu-latest
+    environment:
+      name: issues
     steps:
-      - uses: danhellem/github-actions-issue-to-work-item@v2.2
+      # Auth using Azure Service Principals was added as a part of v2.3
+      # reference: https://github.com/danhellem/github-actions-issue-to-work-item/pull/143
+      - name: Login to Azure
+        uses: azure/login@v2
+        with:
+          client-id: ${{ vars.AZURE_SP_DEVOPS_SYNC_CLIENT_ID }}
+          tenant-id: ${{ vars.AZURE_SP_DEVOPS_SYNC_TENANT_ID }}
+          allow-no-subscriptions: true
+      - name: Get Azure DevOps token
+        id: get_ado_token
+        run:
+          # The resource ID for Azure DevOps is always 499b84ac-1321-427f-aa17-267ca6975798
+          # https://learn.microsoft.com/azure/devops/integrate/get-started/authentication/service-principal-managed-identity
+          echo "ADO_TOKEN=$(az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query "accessToken" --output tsv)" >> $GITHUB_ENV
+      - name: Sync issue to Azure DevOps
+        uses: danhellem/github-actions-issue-to-work-item@v2.3
         env:
-          ado_token: "${{ secrets.ADO_AOCTO_BOT_TOKEN }}"
-          github_token: "${{ secrets.GH_RAD_CI_BOT_PAT }}"
-          ado_organization: "azure-octo"
-          ado_project: "Incubations"
+          ado_token: ${{ env.ADO_TOKEN }}
+          github_token: '${{ secrets.GH_RAD_CI_BOT_PAT }}'
+          ado_organization: 'azure-octo'
+          ado_project: 'Incubations'
           ado_area_path: "Incubations\\Radius"
           ado_iteration_path: "Incubations\\Radius"
-          ado_wit: "GitHub Issue"
-          ado_new_state: "New"
-          ado_active_state: "Active"
-          ado_close_state: "Closed"
-          ado_wit: "GitHub Issue"
+          ado_new_state: 'New'
+          ado_active_state: 'Active'
+          ado_close_state: 'Closed'
+          ado_wit: 'GitHub Issue'
diff --git a/.github/workflows/purge-aws-eks-clusters.yaml b/.github/workflows/purge-aws-eks-clusters.yaml
@@ -0,0 +1,42 @@
+name: Purge AWS EKS Clusters
+
+on:
+  schedule:
+    # Runs every day at 7 AM
+    - cron: "0 7 * * *"
+
+env:
+  GH_TOKEN: ${{ github.token }}
+  AWS_REGION: us-west-2
+
+jobs:
+  purge_eks_clusters:
+    name: Purge AWS EKS Clusters
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install AWS CLI
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y awscli
+
+      - name: Install eksctl
+        run: |
+          curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
+          sudo mv /tmp/eksctl /usr/local/bin
+
+      - name: Delete old EKS clusters
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ env.AWS_REGION }}
+        run: bash .github/scripts/purge-aws-eks-clusters.sh
+
+      - name: Create GitHub issue on failure
+        if: failure() && github.event_name != 'pull_request'
+        run: |
+          gh issue create --title "Purge AWS EKS Clusters workflow failed" \
+            --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." \
+            --repo ${{ github.repository }}
diff --git a/.github/workflows/purge-aws-rds-snapshots.yaml b/.github/workflows/purge-aws-rds-snapshots.yaml
@@ -4,8 +4,8 @@ on:
     # Runs at 00:30 and 12:30
     - cron: "30 0,12 * * *"
 env:
-    GH_TOKEN: ${{ github.token }}
-    AWS_REGION: us-west-2
+  GH_TOKEN: ${{ github.token }}
+  AWS_REGION: us-west-2
 jobs:
   purge_rds_snapshots:
     name: Purge AWS RDS DBInstance snapshots

diff --git a/.github/workflows/purge-test-resources.yaml b/.github/workflows/purge-test-resources.yaml
@@ -5,10 +5,11 @@ on:
 env:
   AZURE_RG_DELETE_LIST_FILE: "az_rg_list.txt"
   VALID_RESOURCE_WINDOW: 6*60*60
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
   purge_azure_resources:
     name: Azure resources clean-ups
-    runs-on: [self-hosted, 1ES.Pool=1ES-Radius]
+    runs-on: [ubuntu-latest]
     steps:
       - name: Login to Azure
         run: |
@@ -22,13 +23,21 @@ jobs:
       - name: List Test Resource Groups
         run: |
           echo "## Test resource group list" >> $GITHUB_STEP_SUMMARY
-          az group list --query "[?starts_with(name, 'samplestest-')].{Name:name, creationTime:tags.creationTime}" -o json > resource_groups.json
+          az group list --query "[?starts_with(name, 'rg-samplestest-')].{Name:name, creationTime:tags.creationTime}" -o json > resource_groups.json
 
           current_time=$(date +%s)
           hours_ago=$((current_time - ${{ env.VALID_RESOURCE_WINDOW }}))
 
-          jq -r '.[] | select(.creationTime == null || .creationTime < '$hours_ago') | .Name' resource_groups.json > ${{ env.AZURE_RG_DELETE_LIST_FILE}}
-          jq -r '.[] | {name: .Name, creationTime: .creationTime // "None"}' resource_groups.json > $GITHUB_STEP_SUMMARY
+          # This jq command processes the data in 'resource_groups.json'.
+          # For each object in the JSON array, it checks if the 'creationTime' property is null or less than the value of 'hours_ago'.
+          # Note that the 'creationTime' is converted to a number for the comparison.
+          # Then the name of the resource group is written to the file specified by the 'AZURE_RG_DELETE_LIST_FILE' environment variable.
+          jq -r '.[] | select(.creationTime == null // (.creationTime | tonumber) < '$hours_ago') | .Name' resource_groups.json > ${{ env.AZURE_RG_DELETE_LIST_FILE}}
+
+          # This jq command also processes the data in 'resource_groups.json'.
+          # For each object in the JSON array, it outputs a string containing the 'Name' property, and the 'creationTime' property.
+          # If 'creationTime' is null, it outputs the string "None" instead.
+          jq -r '.[] | "\(.Name) \(.creationTime // "None")"' resource_groups.json > $GITHUB_STEP_SUMMARY
 
       - name: Delete Azure Resource Groups
         run: |
@@ -40,8 +49,8 @@ jobs:
           done
 
       - name: Create GitHub issue on failure
-        if: ${{ failure() }}
+        if: failure()
         run: |
-          gh issue create --title "Samples purge test resources failed \
+          gh issue create --title "Samples purge test resources failed" \
             --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." \
-            s--repo ${{ github.repository }}
+            --repo ${{ github.repository }}