Merge pull request #640 from microsoft/psl-bug-21342

fix: Sample data processing script fails when using cross-subscription existing AI project resource ID

Author: Prajwal-Microsoft

.github/workflows/CAdeploy.yml

@@ -7,7 +7,7 @@ on:
       - dev
       - demo
   schedule:
-    - cron: '0 6,18 * * *'  # Runs at 6:00 AM and 6:00 PM GMT
+    - cron: "0 6,18 * * *" # Runs at 6:00 AM and 6:00 PM GMT
 
 env:
   GPT_MIN_CAPACITY: 200
@@ -29,7 +29,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      
       - name: Install ODBC Driver 18 for SQL Server
         run: |
           curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
@@ -81,7 +80,6 @@ jobs:
 
       - name: Install Bicep CLI
         run: az bicep install
-        
       - name: Set Deployment Region
         id: set_region
         run: |
@@ -98,7 +96,6 @@ jobs:
           UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
           echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
           echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"
-      
       - name: Check and Create Resource Group
         id: check_create_rg
         run: |
@@ -114,7 +111,6 @@ jobs:
           fi
           # Set output for other jobs
           echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
-      
       - name: Generate Unique Solution Prefix
         id: generate_solution_prefix
         run: |
@@ -137,7 +133,6 @@ jobs:
           else TAG="latest"; fi
           echo "IMAGE_TAG=$TAG" >> $GITHUB_ENV
           echo "Image Tag: $TAG"
-      
       - name: Deploy and extract values from deployment output
         id: get_output
         run: |
@@ -152,16 +147,14 @@ jobs:
             --parameters aiDeploymentsLocation=${{ env.AZURE_LOCATION }} environmentName=${{ env.SOLUTION_PREFIX }} cosmosLocation=westus gptDeploymentCapacity=${{ env.GPT_MIN_CAPACITY }} embeddingDeploymentCapacity=${{ env.TEXT_EMBEDDING_MIN_CAPACITY }} imageTag=${{ env.IMAGE_TAG }} \
             --query "properties.outputs" -o json)
 
-          
-
           echo "Deployment output: $DEPLOY_OUTPUT"
           if [[ -z "$DEPLOY_OUTPUT" ]]; then
             echo "Error: Deployment output is empty. Please check the deployment logs."
             exit 1
           fi
 
-          export AI_FOUNDARY_NAME=$(echo "$DEPLOY_OUTPUT" | jq -r '.aI_FOUNDRY_NAME.value')
-          echo "AI_FOUNDARY_NAME=$AI_FOUNDARY_NAME" >> $GITHUB_ENV
+          export AI_FOUNDRY_RESOURCE_ID=$(echo "$DEPLOY_OUTPUT" | jq -r '.AI_FOUNDRY_RESOURCE_ID.value')
+          echo "AI_FOUNDRY_RESOURCE_ID=$AI_FOUNDRY_RESOURCE_ID" >> $GITHUB_ENV
           export SEARCH_SERVICE_NAME=$(echo "$DEPLOY_OUTPUT" | jq -r '.aI_SEARCH_SERVICE_NAME.value')
           echo "SEARCH_SERVICE_NAME=$SEARCH_SERVICE_NAME" >> $GITHUB_ENV
           export COSMOS_DB_ACCOUNT_NAME=$(echo "$DEPLOY_OUTPUT" | jq -r '.cosmosdB_ACCOUNT_NAME.value')
@@ -182,8 +175,6 @@ jobs:
           echo "CLIENT_NAME=$CLIENT_NAME" >> $GITHUB_ENV
           export RG_NAME=$(echo "$DEPLOY_OUTPUT" | jq -r '.resourcE_GROUP_NAME.value')
           echo "RG_NAME=$RG_NAME" >> $GITHUB_ENV
-          export RESOURCE_GROUP_NAME_FOUNDRY=$(echo "$DEPLOY_OUTPUT" | jq -r '.resourcE_GROUP_NAME_FOUNDRY.value')
-          echo "RESOURCE_GROUP_NAME_FOUNDRY=$RESOURCE_GROUP_NAME_FOUNDRY" >> $GITHUB_ENV
           WEBAPP_URL=$(echo $DEPLOY_OUTPUT | jq -r '.weB_APP_URL.value')
           echo "WEBAPP_URL=$WEBAPP_URL" >> $GITHUB_OUTPUT
           WEB_APP_NAME=$(echo $DEPLOY_OUTPUT | jq -r '.weB_APP_NAME.value')
@@ -195,9 +186,7 @@ jobs:
 
           sleep 30
 
-
-
-      - name: Deploy Infra and Import Sample Data 
+      - name: Deploy Infra and Import Sample Data
         run: |
           set -e
           az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
@@ -221,9 +210,8 @@ jobs:
             "${{ secrets.AZURE_CLIENT_ID }}" \
             "${{ env.RG_NAME }}" \
             "${{ env.SQL_SERVER_NAME }}" \
-            "${{ env.AI_FOUNDARY_NAME }}" \
             "${{ env.SEARCH_SERVICE_NAME }}" \
-            "${{ env.RESOURCE_GROUP_NAME_FOUNDRY }}"
+            "${{ env.AI_FOUNDRY_RESOURCE_ID}}"
 
 
           user_roles_json='[
@@ -236,9 +224,7 @@ jobs:
             "${{ env.SQL_DATABASE }}" \
             "$user_roles_json" \
             "${{ secrets.AZURE_CLIENT_ID }}"
-          
           echo "=== Post-Deployment Script Completed Successfully ==="
-            
 
       - name: Get AI Services name and store in variable
         if: always() && steps.check_create_rg.outcome == 'success'
@@ -262,7 +248,6 @@ jobs:
         run: |
           set -e
           echo "Listing all KeyVaults in the resource group ${{ env.RESOURCE_GROUP_NAME }}..."
-          
           # Get the list of KeyVaults in the specified resource group
           keyvaults=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[?type=='Microsoft.KeyVault/vaults'].name" -o tsv)
 
@@ -288,7 +273,6 @@ jobs:
             # Output the formatted array and save it to the job output
             echo "KEYVAULTS=$keyvault_array" >> $GITHUB_OUTPUT
           fi
-          
       - name: Set Deployment Status
         id: deployment_status
         if: always()
@@ -312,7 +296,7 @@ jobs:
     secrets: inherit
 
   cleanup:
-    if: always() 
+    if: always()
     needs: [deploy, e2e-test]
     runs-on: ubuntu-latest
     env:
@@ -336,12 +320,10 @@ jobs:
           set -e  
           echo "Checking if resource group exists..."
           echo "Resource group name: ${{ env.RESOURCE_GROUP_NAME }}"
-          
           if [ -z "${{ env.RESOURCE_GROUP_NAME }}" ]; then
             echo "Resource group name is empty. Skipping deletion."
             exit 0
           fi
-          
           rg_exists=$(az group exists --name "${{ env.RESOURCE_GROUP_NAME }}")
           if [ "$rg_exists" = "true" ]; then
             echo "Resource group exists. Cleaning..."
@@ -368,12 +350,9 @@ jobs:
 
           # Remove the surrounding square brackets and quotes, if they exist
           stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g' | sed 's/"//g')
-          
           # Convert the comma-separated string into an array
           IFS=',' read -r -a resources_to_check <<< "$stripped_keyvaults"
-          
           echo "List of resources to check: ${resources_to_check[@]}"
-         
           # Check if resource group still exists before listing resources
           rg_exists=$(az group exists --name "${{ env.RESOURCE_GROUP_NAME }}")
           if [ "$rg_exists" = "false" ]; then
@@ -472,7 +451,6 @@ jobs:
 
           # Remove the surrounding square brackets and quotes, if they exist
           stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g' | sed 's/"//g')
-          
           # Convert the comma-separated string into an array
           IFS=',' read -r -a keyvault_array <<< "$stripped_keyvaults"
 
@@ -506,15 +484,13 @@ jobs:
         if: failure() || needs.deploy.result == 'failure' || needs.e2e-test.result == 'failure'
         run: |
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-      
           # Construct the email body
           EMAIL_BODY=$(cat <<EOF
           {
             "body": "<p>Dear Team,</p><p>We would like to inform you that the Build-your-own-copilot-Solution-Accelerator(Client Advisior) Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
           }
           EOF
           )
-      
           # Send the notification
           curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
             -H "Content-Type: application/json" \

docs/DeploymentGuide.md

@@ -143,6 +143,13 @@ To adjust quota settings, follow these [steps](./AzureGPTQuotaSettings.md).
 
   Guide to get your [Existing Workspace ID](/docs/re-use-log-analytics.md)
 
+</details>
+<details>
+
+  <summary><b>Reusing an Existing Azure AI Foundry Project</b></summary>
+
+  Guide to get your [Existing Project ID](/docs/re-use-foundry-project.md)
+
 </details>
 
 ### Deploying with AZD

docs/images/re_use_foundry_project/azure_ai_foundry_list.png

@@ -0,0 +1,44 @@
+[← Back to *DEPLOYMENT* guide](/docs/DeploymentGuide.md#deployment-options--steps)
+
+# Reusing an Existing Azure AI Foundry Project
+To configure your environment to use an existing Azure AI Foundry Project, follow these steps:
+---
+### 1. Go to Azure Portal
+Go to https://portal.azure.com
+
+### 2. Search for Azure AI Foundry
+In the search bar at the top, type "Azure AI Foundry" and click on it. Then select the Foundry service instance where your project exists.
+
+![alt text](../docs/images/re_use_foundry_project/azure_ai_foundry_list.png)
+
+### 3. Navigate to Projects under Resource Management
+On the left sidebar of the Foundry service blade:
+
+- Expand the Resource Management section
+- Click on Projects (this refers to the active Foundry project tied to the service)
+
+### 4. Click on the Project
+From the Projects view: Click on the project name to open its details
+
+    Note: You will see only one project listed here, as each Foundry service maps to a single project in this accelerator
+
+![alt text](../docs/images/re_use_foundry_project/navigate_to_projects.png)
+
+### 5. Copy Resource ID
+In the left-hand menu of the project blade: 
+
+- Click on Properties under Resource Management
+- Locate the Resource ID field
+- Click on the copy icon next to the Resource ID value
+
+![alt text](../docs/images/re_use_foundry_project/project_resource_id.png)
+
+### 6. Set the Foundry Project Resource ID in Your Environment
+Run the following command in your terminal
+```bash
+azd env set AZURE_EXISTING_AI_PROJECT_RESOURCE_ID '<Existing AI Project Resource ID>'
+```
+Replace `<Existing AI Project Resource ID>` with the value obtained from Step 5.
+
+### 7. Continue Deployment
+Proceed with the next steps in the [deployment guide](/documents/DeploymentGuide.md#deployment-options--steps).

docs/images/re_use_foundry_project/navigate_to_projects.png

@@ -146,6 +146,11 @@ resource aiFoundryProject 'Microsoft.CognitiveServices/accounts/projects@2025-04
   }
 }
 
+resource existingAiFoundry 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = if (!empty(azureExistingAIProjectResourceId)) {
+  name: existingAIFoundryName
+  scope: resourceGroup(existingAIServiceSubscription, existingAIServiceResourceGroup)
+}
+
 @batchSize(1)
 resource aiFModelDeployments 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = [
   for aiModeldeployment in aiModelDeployments: if (empty(azureExistingAIProjectResourceId)) {
@@ -228,8 +233,7 @@ resource cognitiveServicesOpenAIUser 'Microsoft.Authorization/roleDefinitions@20
   name: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd'
 }
 
-
-resource assignOpenAIRoleToAISearch 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (empty(azureExistingAIProjectResourceId))  {
+resource assignOpenAIRoleToAISearch 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (empty(azureExistingAIProjectResourceId)) {
   name: guid(resourceGroup().id, aiFoundry.id, cognitiveServicesOpenAIUser.id)
   scope: aiFoundry
   properties: {
@@ -247,7 +251,7 @@ module assignOpenAIRoleToAISearchExisting 'deploy_foundry_model_role_assignment.
     roleAssignmentName: guid(resourceGroup().id, aiSearch.id, cognitiveServicesOpenAIUser.id, 'openai-foundry')
     aiFoundryName: !empty(azureExistingAIProjectResourceId) ? existingAIFoundryName : aiFoundryName
     principalId: aiSearch.identity.principalId
-aiProjectName: !empty(azureExistingAIProjectResourceId) ? existingAIProjectName : aiProjectName
+    aiProjectName: !empty(azureExistingAIProjectResourceId) ? existingAIProjectName : aiProjectName
     aiModelDeployments: aiModelDeployments
   }
 }
@@ -377,6 +381,7 @@ output aoaiEndpoint string = !empty(existingOpenAIEndpoint)
   ? existingOpenAIEndpoint
   : aiFoundry.properties.endpoints['OpenAI Language Model Instance API'] //aiServices_m.properties.endpoint
 output aiFoundryName string = !empty(existingAIFoundryName) ? existingAIFoundryName : aiFoundryName //aiServicesName_m
+output aiFoundryId string = !empty(azureExistingAIProjectResourceId) ? existingAiFoundry.id : aiFoundry.id
 
 output aiSearchName string = aiSearchName
 output aiSearchId string = aiSearch.id

docs/images/re_use_foundry_project/project_resource_id.png

@@ -55,12 +55,21 @@ param embeddingDeploymentCapacity int = 80
 param imageTag string = 'latest'
 
 //restricting to these regions because assistants api for gpt-4o-mini is available only in these regions
-@allowed(['australiaeast','eastus', 'eastus2','francecentral','japaneast','swedencentral','uksouth', 'westus', 'westus3'])
+@allowed([
+  'australiaeast'
+  'eastus'
+  'eastus2'
+  'francecentral'
+  'japaneast'
+  'swedencentral'
+  'uksouth'
+  'westus'
+  'westus3'
+])
 // @description('Azure OpenAI Location')
 // param AzureOpenAILocation string = 'eastus2'
-
 @metadata({
-  azd:{
+  azd: {
     type: 'location'
     usageName: [
       'OpenAI.GlobalStandard.gpt-4o-mini,200'
@@ -112,7 +121,6 @@ var useInternalStream = 'True'
 var useAIProjectClientFlag = 'False'
 var sqlServerFqdn = '${sqlDBModule.outputs.sqlServerName}.database.windows.net'
 
-
 var functionAppSqlPrompt = '''Generate a valid T-SQL query to find {query} for tables and columns provided below:
    1. Table: Clients
    Columns: ClientId, Client, Email, Occupation, MaritalStatus, Dependents
@@ -308,12 +316,11 @@ output STORAGE_CONTAINER_NAME string = storageAccountModule.outputs.storageConta
 output KEY_VAULT_NAME string = keyvaultModule.outputs.keyvaultName
 output COSMOSDB_ACCOUNT_NAME string = cosmosDBModule.outputs.cosmosAccountName
 output RESOURCE_GROUP_NAME string = resourceGroup().name
-output RESOURCE_GROUP_NAME_FOUNDRY string = aifoundry.outputs.resourceGroupNameFoundry
+output AI_FOUNDRY_RESOURCE_ID string = aifoundry.outputs.aiFoundryId
 output SQLDB_SERVER_NAME string = sqlDBModule.outputs.sqlServerName
 output SQLDB_DATABASE string = sqlDBModule.outputs.sqlDbName
 output MANAGEDIDENTITY_WEBAPP_NAME string = managedIdentityModule.outputs.managedIdentityWebAppOutput.name
 output MANAGEDIDENTITY_WEBAPP_CLIENTID string = managedIdentityModule.outputs.managedIdentityWebAppOutput.clientId
-output AI_FOUNDRY_NAME string = aifoundry.outputs.aiFoundryName
 output AI_SEARCH_SERVICE_NAME string = aifoundry.outputs.aiSearchService
 output WEB_APP_NAME string = appserviceModule.outputs.webAppName
 output APP_ENV string = appEnvironment

docs/re-use-foundry-project.md

@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.36.177.2456",
-      "templateHash": "8388189115253050142"
+      "templateHash": "11128400850485102096"
     }
   },
   "parameters": {
@@ -772,7 +772,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.36.177.2456",
-              "templateHash": "5605479917210969670"
+              "templateHash": "16824105341593554305"
             }
           },
           "parameters": {
@@ -1612,6 +1612,10 @@
               "type": "string",
               "value": "[if(not(empty(variables('existingAIFoundryName'))), variables('existingAIFoundryName'), variables('aiFoundryName'))]"
             },
+            "aiFoundryId": {
+              "type": "string",
+              "value": "[if(not(empty(parameters('azureExistingAIProjectResourceId'))), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', variables('existingAIServiceSubscription'), variables('existingAIServiceResourceGroup')), 'Microsoft.CognitiveServices/accounts', variables('existingAIFoundryName')), resourceId('Microsoft.CognitiveServices/accounts', variables('aiFoundryName')))]"
+            },
             "aiSearchName": {
               "type": "string",
               "value": "[variables('aiSearchName')]"
@@ -3136,9 +3140,9 @@
       "type": "string",
       "value": "[resourceGroup().name]"
     },
-    "RESOURCE_GROUP_NAME_FOUNDRY": {
+    "AI_FOUNDRY_RESOURCE_ID": {
       "type": "string",
-      "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_ai_foundry'), '2022-09-01').outputs.resourceGroupNameFoundry.value]"
+      "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_ai_foundry'), '2022-09-01').outputs.aiFoundryId.value]"
     },
     "SQLDB_SERVER_NAME": {
       "type": "string",
@@ -3156,10 +3160,6 @@
       "type": "string",
       "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_managed_identity'), '2022-09-01').outputs.managedIdentityWebAppOutput.value.clientId]"
     },
-    "AI_FOUNDRY_NAME": {
-      "type": "string",
-      "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_ai_foundry'), '2022-09-01').outputs.aiFoundryName.value]"
-    },
     "AI_SEARCH_SERVICE_NAME": {
       "type": "string",
       "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_ai_foundry'), '2022-09-01').outputs.aiSearchService.value]"