diff --git a/.pipelines/run-pipeline.yaml b/.pipelines/run-pipeline.yaml index 48e5c868e0..3056bc85a7 100644 --- a/.pipelines/run-pipeline.yaml +++ b/.pipelines/run-pipeline.yaml @@ -16,10 +16,9 @@ parameters: type: string default: '' - stages: - stage: setup - displayName: ACN + displayName: "[PreBuild] Configure and Provision Required Resources" # Block build start until pre-build validation occurs. dependsOn: pre_build variables: @@ -85,7 +84,6 @@ stages: workingDirectory: $(ACN_DIR) - script: | - mkdir -p ./output/bins cd ./output find . -name '*.tgz' -print -exec mv -t ./bins/ {} + find . -name '*.zip' -print -exec mv -t ./bins/ {} + @@ -93,13 +91,14 @@ stages: rm -rf !("bins") name: "PrepareArtifacts" displayName: "Prepare Artifacts" - + - task: CopyFiles@2 inputs: sourceFolder: "output" targetFolder: $(Build.ArtifactStagingDirectory) condition: succeeded() + - stage: containerize displayName: Build Images dependsOn: @@ -157,23 +156,6 @@ stages: name: cni os: windows os_version: ltsc2025 - cni_dropgz_linux_amd64: - Suffix: cni_dropgz_linux_amd64 - arch: amd64 - name: cni-dropgz - os: linux - cni_dropgz_windows2019_amd64: - Suffix: cni_dropgz_windows2019_amd64 - arch: amd64 - name: cni-dropgz - os: windows - os_version: ltsc2019 - cni_dropgz_windows2022_amd64: - Suffix: cni_dropgz_windows2022_amd64 - arch: amd64 - name: cni-dropgz - os: windows - os_version: ltsc2022 cns_linux_amd64: Suffix: cns_linux_amd64 arch: amd64 @@ -247,11 +229,6 @@ stages: name: cni os: linux Suffix: cni-linux-arm64 - cni_dropgz_linux_arm64: - arch: arm64 - name: cni-dropgz - os: linux - Suffix: cni-dropgz cns_linux_arm64: arch: arm64 name: cns @@ -310,11 +287,6 @@ stages: os_versions: ltsc2019 ltsc2022 ltsc2025 platforms: linux/amd64 linux/arm64 windows/amd64 Suffix: cni - cni_dropgz: - name: cni-dropgz - os_versions: ltsc2019 ltsc2022 - platforms: linux/amd64 linux/arm64 windows/amd64 - Suffix: cni-dropgz cns: name: cns os_versions: ltsc2019 ltsc2022 ltsc2025 diff --git a/.pipelines/templates/artifact-storage.steps.yaml b/.pipelines/templates/artifact-storage.steps.yaml new file mode 100644 index 0000000000..26bd94b591 --- /dev/null +++ b/.pipelines/templates/artifact-storage.steps.yaml @@ -0,0 +1,536 @@ + +steps: + +- task: AzureCLI@2 + name: build + displayName: "[Output] SC Environment App Details" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + # App OID + SP_APP_OID=$(az ad sp show --id "$servicePrincipalId" | jq -rc '.id') + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_SP_APP_OID;isoutput=true;]$SP_APP_OID" + # Get Subscription ID. + SUBSCRIPTION_ID=$(az account show | jq -rc '.id') + SUBSCRIPTION_NAME=$(az account show | jq -rc '.name') + TENANT_ID=$(az account show | jq -rc '.tenantId') + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_SUBSCRIPTION_ID;isoutput=true;issecret=true]$SUBSCRIPTION_ID" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_SUBSCRIPTION_NAME;isoutput=true;issecret=true]$SUBSCRIPTION_NAME" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_TENANT_ID;isoutput=true;issecret=true]$TENANT_ID" + + +## Resource Groups ## +- bash: | + UNIQUE_ID=`tr -dc '0-9a-z' < /dev/urandom | head -c${1:-7}` + echo >&2 "##vso[task.setvariable variable=LOCAL_ACNCI_UNIQUE_ID]$UNIQUE_ID" + displayName: "[Infra] Generate Unique ID" + +- template: get-resources.steps.yaml + parameters: + resourceType: resourcegroups + serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + inputs: + buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + +- template: create-or-update-resource.steps.yaml + parameters: + serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + resourceType: resourcegroups + refreshAfterCreation: True + createCondition: | + and(succeeded(), + or(not(variables.OUT_RESULT_LENGTH), + lt(variables.OUT_RESULT_LENGTH, variables.ACNCI_RG_POOL_SIZE))) + updateCondition: False + inputs: + resourceGroupList: $(OUT_RESULT) + resourceGroupCount: $(OUT_RESULT_LENGTH) + resourceGroupName: $(ACNCI_RG_PREFIX)$(LOCAL_ACNCI_UNIQUE_ID) + resourceGroupLocation: $(ACNCI_RG_LOCATION) + buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + +- task: AzureCLI@2 + name: resourcegroups + displayName: "[Output] Build Resource Group" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + RANDOM_SELECT=`tr -dc '1-9' < /dev/urandom | head -c${1:-7}` + IDX=$(( "$RANDOM_SELECT" % "$INFRA_RG_LENGTH" )) + RG_DATA=$(echo "$INFRA_RG_LIST" | jq --argjson IDX "$IDX" -rc '.[$IDX]') + RG_NAME=$(echo "$RG_DATA" | jq -rc '.name') + RG_ID=$(echo "$RG_DATA" | jq -rc '.id') + + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_RESOURCEGROUP;isoutput=true;]$RG_NAME" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_RESOURCEGROUP_LOCATION;isoutput=true;]$ACNCI_RG_LOCATION" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILD_RESOURCEGROUP_ID;isoutput=true;]$RG_ID" + env: + ACNCI_RG_LOCATION: $(ACNCI_RG_LOCATION) + INFRA_RG_LIST: $(OUT_RESULT) + INFRA_RG_LENGTH: $(OUT_RESULT_LENGTH) + + +## MI Service Connection + +# SERVICECONNECTION_PRINCIPALID: ${{ parameters.inputs.serviceConnectionPrincipalId }} +# SUBSCRIPTION_ID: ${{ parameters.inputs.subscriptionId }} +# SUBSCRIPTION_NAME: ${{ parameters.inputs.subscriptionName }} +# SERVICECONNECTION_TENANTID: ${{ parameters.inputs.tenantId }} +# SERVICECONNECTION_NAME: ${{ parameters.inputs.serviceConnectionName }} +# +#- template: create-or-update-resource.steps.yaml +# parameters: +# resourceType: serviceconnection +# serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) +# createCondition: | +# and(succeeded(), +# or(not(variables.OUT_RESULT_LENGTH), +# eq(variables.OUT_RESULT_LENGTH, 'null'), +# lt(variables.OUT_RESULT_LENGTH, 1))) +# updateCondition: False +# inputs: +# serviceConnectionName: $(managedidentity.ACNCI_MANAGEDIDENTITY_NAME)-serviceconnection +# serviceConnectionPrincipalId: $(managedidentity.ACNCI_MANAGEDIDENTITY_OBJECTID) +# subscriptionId: $(build.ACNCI_BUILD_SUBSCRIPTION_ID) +# subscriptionName: $(build.ACNCI_BUILD_SUBSCRIPTION_NAME) +# tenantId: $(build.ACNCI_BUILD_TENANT_ID) +# buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) +# buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) +# buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYBUILDID) +# + +## MI Role Definition ## + +#- template: get-resources.steps.yaml + #parameters: + #resourceType: roledefinition + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #inputs: + #roleName: $(ACNCI_BUILDUSER_ROLE_NAME) + #roleDefinitionFileLocation: ./azure-container-networking/.pipelines/templates/mi-build-role.json + #subscriptionId: $(build.ACNCI_BUILD_SUBSCRIPTION_ID) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) +# +#- template: create-or-update-resource.steps.yaml + #parameters: + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #resourceType: roledefinition + #refreshAfterCreation: True + #createCondition: | + #and(succeeded(), + #or(not(variables.OUT_RESULT_LENGTH), + #lt(variables.OUT_RESULT_LENGTH, 1))) + #updateCondition: | + #and(succeeded(), + #gt(variables.OUT_RESULT_LENGTH, 0)) + #inputs: + #roleName: $(ACNCI_BUILDUSER_ROLE_NAME) + #roleDefinitionJson: $(OUT_RESULT) + #roleDefinitionFileLocation: ./azure-container-networking/.pipelines/templates/mi-build-role.json + #resourceGroupId: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #subscriptionId: $(build.ACNCI_BUILD_SUBSCRIPTION_ID) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + #buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) +# +# +### Provision MI Roles +# +#- template: get-resources.steps.yaml + #parameters: + #resourceType: roleassignments + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #inputs: + #roleName: $(ACNCI_BUILDUSER_ROLE_NAME) + #managedIdentityObjectId: $(build.ACNCI_BUILD_SP_APP_OID) + #resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) +# +#- template: create-or-update-resource.steps.yaml + #parameters: + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #resourceType: roleassignments + #createCondition: | + #and(succeeded(), + #or(not(variables.OUT_RESULT_LENGTH), + #eq(variables.OUT_RESULT, 'null'), + #lt(variables.OUT_RESULT_LENGTH, 1))) + #updateCondition: False + #inputs: + #roleName: $(ACNCI_BUILDUSER_ROLE_NAME) + #managedIdentityObjectId: $(build.ACNCI_BUILD_SP_APP_OID) + #resourceGroupId: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP_ID) + #resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + #buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + +## Enable Build User Service Connection Access + +- template: get-resources.steps.yaml + parameters: + resourceType: serviceconnection + serviceConnection: $(ACNCI_BUILDUSER_SERVICECONNECTION_NAME) + inputs: + serviceConnectionId: $(ACNCI_BUILDUSER_SERVICECONNECTION_ID) + resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + +- task: AzureCLI@2 + name: serviceconnection + displayName: "[Output] Build User ServiceConnection Details" + inputs: + azureSubscription: $(ACNCI_BUILDUSER_SERVICECONNECTION_NAME) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + # Select MI to use + RANDOM_SELECT=`tr -dc '1-9' < /dev/urandom | head -c${1:-7}` + IDX=$(( "$RANDOM_SELECT" % "$SC_LIST_LENGTH" )) + SC_DATA=$(echo "$SC_LIST" | jq --argjson IDX "$IDX" -rc '.[$IDX]') + + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILDUSER_SERVICECONNECTION_ID;isoutput=true]$ACNCI_BUILDUSER_SERVICECONNECTION_ID" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILDUSER_SERVICECONNECTION_KEY;isoutput=true;issecret=true]$servicePrincipalKey" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILDUSER_SERVICECONNECTION_TENANTID;isoutput=true;issecret=true]$tenantId" + SC_NAME=$(echo "$SC_DATA" | jq -rc '.name') + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILDUSER_SERVICECONNECTION_NAME;isoutput=true]$SC_NAME" + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILDUSER_SERVICECONNECTION_APP_OID;isoutput=true]$ACNCI_BUILDUSER_SERVICECONNECTION_APP_OID" + SC_APPID=$(echo "$SC_DATA" | jq -rc '.authorization.parameters.serviceprincipalid') + echo >&2 "##vso[task.setvariable variable=ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID;isoutput=true]$ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID" + env: + SC_LIST: $(OUT_RESULT) + SC_LIST_LENGTH: $(OUT_RESULT_LENGTH) + +# storage accounts + +- template: get-resources.steps.yaml + parameters: + resourceType: storageaccounts + serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + inputs: + resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYBUILDID) + +- template: create-or-update-resource.steps.yaml + parameters: + serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + resourceType: storageaccounts + refreshAfterCreation: True + createCondition: | + and(succeeded(), + or(not(variables.OUT_RESULT_LENGTH), + lt(variables.OUT_RESULT_LENGTH, 1))) + updateCondition: False + inputs: + storageAccountName: '$(ACNCI_SA_PREFIX)$(LOCAL_ACNCI_UNIQUE_ID)' + storageAccountLocation: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP_LOCATION) + resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + managedIdentityResourceId: $(managedidentity.ACNCI_MANAGEDIDENTITY_ID) + buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYBUILDID) + buildTagBuildUserAppId: $(ACNCI_BUILDTAG_BUILDUSER_APPID) + +- task: AzureCLI@2 + name: artifact_storage + displayName: "[Output] Storage Account Data" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + SA_RANDOM_SELECT=`tr -dc '1-9' < /dev/urandom | head -c${1:-7}` + R_INDEX=$(( "$SA_RANDOM_SELECT" % "$SA_LIST_LENGTH" )) + # Make a random selection from the pool + SA_DATA=$(echo "$SA_LIST" | jq --argjson IDX "$R_INDEX" -rc '.[$IDX]') + if (( "$SA_LIST_LENGTH" > 0 )); then + # Azure Resource ID + SA_ID=$(echo "$SA_DATA" | jq -r '.id') + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_ID;isoutput=true]$SA_ID" + # Azure Resource Name + SA_NAME=$(echo "$SA_DATA" | jq -r '.name') + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_NAME;isoutput=true]$SA_NAME" + # Storage Account Subscription + SA_SUBSCRIPTION=$(echo "$SA_DATA" | jq -r '.subscriptionId') + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_SUBSCRIPTION;isoutput=true]$SA_SUBSCRIPTION" + # Service Connection + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_SERVICECONNECTION;isoutput=true]$SA_SERVICE_CONN" + # Storage Account Resource Group + SA_RG=$(echo "$SA_DATA" | jq -r '.resourceGroup') + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_RESOURCEGROUP;isoutput=true]$SA_RG" + # Storage Account Location + SA_LOCATION=$(echo "$SA_DATA" | jq -r '.location') + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_LOCATION;isoutput=true]$SA_LOCATION" + # Current Build Storage Data + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_CONTAINER_NAME;isoutput=true]$STORAGECONTAINER_NAME" + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT_BLOB_PATH;isoutput=true]$STORAGEBLOB_PATH" + # - Local Use Only - + # SA Object + echo >&2 "##vso[task.setvariable variable=ACNCI_STORAGEACCOUNT]$SA_DATA" + echo $SA_DATA + else + echo >&2 "##[error]No storage accounts available for use." + exit 1 + fi + env: + SA_LIST: $(OUT_RESULT) + SA_LIST_LENGTH: $(OUT_RESULT_LENGTH) + SA_SERVICE_CONN: $(ACN_TEST_SERVICE_CONNECTION) + STORAGECONTAINER_NAME: "azure-container-networking-pr" + STORAGEBLOB_PATH: $(Build.BuildId)/$(System.JobAttempt) + + +- task: AzureCLI@2 + displayName: "[Provision] Give Test Permissions to Primary" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + addSpnToEnvironment: true + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az role assignment create \ + --role "Storage Blob Data Contributor" \ + --assignee-object-id "$servicePrincipalId" \ + --assignee-principal-type "ServicePrincipal" \ + --scope "$RESOURCEGROUP_ID" + env: + RESOURCEGROUP_ID: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP_ID) + + +- task: AzureCLI@2 + name: build_storage + displayName: "[Provision] Establish Build Storage" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az storage container create \ + --account-name "$STORAGEACCOUNT_NAME" \ + --resource-group "$RESOURCEGROUP_NAME" \ + --name "$STORAGECONTAINER_NAME" \ + --auth-mode login || echo >&2 "##[info]Storage container provisioned." + + echo "hold" | az storage blob upload \ + --data @- \ + --name "$STORAGEBLOB_PATH/.created" \ + --container-name "$STORAGECONTAINER_NAME" \ + --account-name "$STORAGEACCOUNT_NAME" \ + --overwrite \ + --auth-mode login + env: + RESOURCEGROUP_NAME: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + STORAGEACCOUNT_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_NAME) + STORAGECONTAINER_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_CONTAINER_NAME) + STORAGEBLOB_PATH: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID)/$(Build.BuildId)/$(System.JobAttempt) + + +## Managed Identity ## + +#- template: get-resources.steps.yaml + #parameters: + #resourceType: managedidentity + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #inputs: + #resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) +# +#- template: create-or-update-resource.steps.yaml + #parameters: + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #resourceType: managedidentity + #refreshAfterCreation: True + #createCondition: | + #and(succeeded(), + #or(not(variables.OUT_RESULT_LENGTH), + #eq(variables.OUT_RESULT_LENGTH, 'null'), + #lt(variables.OUT_RESULT_LENGTH, 1))) + #updateCondition: False + #inputs: + #managedIdentityList: $(OUT_RESULT) + #managedIdentityCount: $(OUT_RESULT_LENGTH) + #managedIdentityName: '$(ACNCI_MANAGEDIDENTITY_PREFIX)$(LOCAL_ACNCI_UNIQUE_ID)-$(resourcegroups.ACNCI_BUILD_RESOURCEGROUP_LOCATION)' + #managedIdentityLocation: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP_LOCATION) + #resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + #buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYBUILDID) +# +#- task: AzureCLI@2 + #name: managedidentity + #displayName: "[Output] Build User ManagedIdentity Details" + #inputs: + #azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + #scriptType: bash + #scriptLocation: inlineScript + #addSpnToEnvironment: true + #inlineScript: | + #set -e + #[[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x +# + ## Select MI to use + #RANDOM_SELECT=`tr -dc '1-9' < /dev/urandom | head -c${1:-7}` + #IDX=$(( "$RANDOM_SELECT" % "$MI_LIST_LENGTH" )) + #MI_DATA=$(echo "$MI_LIST" | jq --argjson IDX "$IDX" -rc '.[$IDX]') +# + #MI_ID=$(echo "$MI_DATA" | jq -r '.id') + #echo >&2 "##vso[task.setvariable variable=ACNCI_MANAGEDIDENTITY_ID;isoutput=true]$MI_ID" + #MI_PRINCIPALID=$(echo "$MI_DATA" | jq -r '.principalId') + #echo >&2 "##vso[task.setvariable variable=ACNCI_MANAGEDIDENTITY_OBJECTID;isoutput=true]$MI_PRINCIPALID" + #MI_APPID=$(echo "$MI_DATA" | jq -r '.clientId') + #echo >&2 "##vso[task.setvariable variable=ACNCI_MANAGEDIDENTITY_APPID;isoutput=true]$MI_APPID" + #MI_NAME=$(echo "$MI_DATA" | jq -r '.name') + #echo >&2 "##vso[task.setvariable variable=ACNCI_MANAGEDIDENTITY_NAME;isoutput=true]$MI_NAME" + #env: + #ACNCI_BUILD_RESOURCEGROUP: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #MI_LIST: $(OUT_RESULT) + #MI_LIST_LENGTH: $(OUT_RESULT_LENGTH) + +- task: AzureCLI@2 + displayName: "[Test] Blob Access Should Fail" + continueOnError: true + inputs: + azureSubscription: $(ACNCI_BUILDUSER_SERVICECONNECTION_NAME) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -x + az storage blob download \ + --auth-mode login \ + --container-name "$STORAGECONTAINER_NAME" \ + --account-name "$STORAGEACCOUNT_NAME" \ + --name "$STORAGEBLOB_PATH/.created" \ + --file output + cat ./output + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + STORAGEACCOUNT_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_NAME) + STORAGECONTAINER_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_CONTAINER_NAME) + STORAGEBLOB_PATH: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID)/$(Build.BuildId)/$(System.JobAttempt) + SP_ID: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID) + SP_TENANT: $(build.ACNCI_BUILD_TENANT_ID) + +- task: AzureCLI@2 + displayName: "[Provision] Build User Access Permissions" + continueOnError: true + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + az upgrade --yes + + CONDITION="(@Resource[Microsoft.Storage/storageAccounts/blobServices/containers:Name] stringEquals '$STORAGECONTAINER_NAME') AND (@Resource[Microsoft.Storage/storageAccounts/blobServices/containers/blobs:Name] stringStartsWith '$EXPECTED_STORAGEBLOB_PREFIX')" + az role assignment create \ + --role "Storage Blob Data Contributor" \ + --assignee-object-id "$SP_ID" \ + --assignee-principal-type ServicePrincipal \ + --scope "$STORAGEACCOUNT_ID/blobServices/default/containers/$STORAGECONTAINER_NAME" \ + --condition "$CONDITION" \ + --condition-version '2.0' + env: + STORAGEACCOUNT_ID: $(artifact_storage.ACNCI_STORAGEACCOUNT_ID) + STORAGEACCOUNT_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_NAME) + STORAGECONTAINER_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_CONTAINER_NAME) + EXPECTED_STORAGEBLOB_PREFIX: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID)/$(Build.BuildId) + SP_ID: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID) + SP_TENANT: $(build.ACNCI_BUILD_TENANT_ID) + +- task: AzureCLI@2 + displayName: "[Test] Blob Access Should Succeed" + continueOnError: true + inputs: + azureSubscription: $(ACNCI_BUILDUSER_SERVICECONNECTION_NAME) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -xe + az storage blob download \ + --auth-mode login \ + --account-name "$STORAGEACCOUNT_NAME" \ + --container-name "$STORAGECONTAINER_NAME" \ + --name "$STORAGEBLOB_PATH/.created" \ + --file output + cat ./output + env: + STORAGEACCOUNT_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_NAME) + STORAGECONTAINER_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_CONTAINER_NAME) + STORAGEBLOB_PATH: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID)/$(Build.BuildId)/$(System.JobAttempt) + SP_ID: $(serviceconnection.ACNCI_BUILDUSER_SERVICECONNECTION_CLIENTID) + SP_TENANT: $(build.ACNCI_BUILD_TENANT_ID) + +#- template: create-or-update-resource.steps.yaml + #parameters: + #serviceConnection: $(ACN_TEST_SERVICE_CONNECTION) + #resourceType: managedidentity + #refreshAfterCreation: True + #createCondition: False + #updateCondition: succeeded() + #inputs: + #managedIdentityName: $(managedidentity.ACNCI_MANAGEDIDENTITY_NAME) + #resourceGroupName: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #buildTagDefinitionIdKey: $(ACNCI_BUILDTAG_DEFINITIONID) + #buildTagCreatedByAppIdKey: $(ACNCI_BUILDTAG_CREATEDBYAPPID) + #buildTagCreatedByBuildIdKey: $(ACNCI_BUILDTAG_CREATEDBYBUILDID) +# +#- task: AzureCLI@2 + #displayName: "[Test] Blob Access" + #continueOnError: true + #inputs: + #azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + #scriptType: bash + #scriptLocation: inlineScript + #addSpnToEnvironment: true + #inlineScript: | + #set +x + #az login --service-principal --username "$MI_NAME" --tenant "$MI_TENANT" + #az storage blob download --auth-mode login --container-name "$STORAGECONTAINER_NAME" --account-name "$STORAGEACCOUNT_NAME" --name "$STORAGEBLOB_PATH/.created" --file output + #cat ./output + #env: + #RESOURCEGROUP_NAME: $(resourcegroups.ACNCI_BUILD_RESOURCEGROUP) + #STORAGEACCOUNT_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_NAME) + #STORAGECONTAINER_NAME: $(artifact_storage.ACNCI_STORAGEACCOUNT_CONTAINER_NAME) + #MANAGEDIDENTITY_OBJECTID: $(managedidentity.ACNCI_MANAGEDIDENTITY_OBJECTID) + #STORAGEBLOB_PATH: $(Build.BuildId)/$(System.JobAttempt) + #MI_NAME: $(managedidentity.ACNCI_MANAGEDIDENTITY_NAME) + #MI_TENANT: $(build.ACNCI_BUILD_TENANT_ID) diff --git a/.pipelines/templates/copy-storage-artifacts.steps.yaml b/.pipelines/templates/copy-storage-artifacts.steps.yaml new file mode 100644 index 0000000000..6fb644d621 --- /dev/null +++ b/.pipelines/templates/copy-storage-artifacts.steps.yaml @@ -0,0 +1,38 @@ +parameters: +- name: sourceFolder + type: string + +- name: targetFolder + type: string + default: $(Build.ArtifactStagingDirectory) + +steps: + +# Dependency Variables: +# IS_TRUE (run-pipeline level pipeline var) +# SYSTEM_DEBUG (global pipeline var) +# SA_NAME (stageDep) +# CONTAINER_NAME (stageDep) +# BLOB_PATH (stageDep) +- task: AzureCLI@2 + displayName: "Upload to Storage" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az storage blob upload-batch \ + --account-name "$SA_NAME" \ + --destination "$CONTAINER_NAME" \ + --source "$SOURCE" \ + --destination-path "$BLOB_PATH" \ + --auth-mode login + env: + SOURCE: ${{ parameters.sourceFolder }} + TARGET: ${{ parameters.targetFolder }} + SA_NAME: $(SA_NAME) + CONTAINER_NAME: $(CONTAINER_NAME) diff --git a/.pipelines/templates/create-or-update-resource.steps.yaml b/.pipelines/templates/create-or-update-resource.steps.yaml new file mode 100644 index 0000000000..8ffd3f9134 --- /dev/null +++ b/.pipelines/templates/create-or-update-resource.steps.yaml @@ -0,0 +1,248 @@ + +parameters: +- name: serviceConnection + type: string + +- name: createCondition + type: object + default: 'succeeded()' + +- name: updateCondition + type: object + default: 'succeeded()' + +- name: refreshAfterCreation + type: boolean + default: False + +- name: resourceType + type: string + values: + - roleassignments + - resourcegroups + - storageaccounts + - roledefinition + - managedidentity + - serviceconnection + +- name: inputs + type: object + default: {} +- name: outputVariableName + type: string + default: 'OUT_RESULT' + +steps: + +- ${{ if parameters.createCondition }}: + - task: AzureCLI@2 + displayName: "[Create] ARM Details - (${{ parameters.resourceType }})" + condition: ${{ parameters.createCondition }} + + env: + BUILDTAG_DEFINITIONID: ${{ parameters.inputs.buildTagDefinitionIdKey }} + BUILDTAG_CREATEDBYAPPID: ${{ parameters.inputs.buildTagCreatedByAppIdKey }} + BUILDTAG_CREATEDBYBUILDID: ${{ parameters.inputs.buildTagCreatedByBuildIdKey }} + ${{ if eq(parameters.resourceType, 'roleassignments') }}: + ROLE_NAME: ${{ parameters.inputs.roleName }} + RESOURCEGROUP_ID: ${{ parameters.inputs.resourceGroupId }} + MANAGEDIDENTITY_OBJECTID: ${{ parameters.inputs.managedIdentityObjectId }} + + ${{ elseif eq(parameters.resourceType, 'resourcegroups') }}: + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + RESOURCEGROUP_LOCATION: ${{ parameters.inputs.resourceGroupLocation }} + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + MANAGEDIDENTITY_ARMID: ${{ parameters.inputs.managedIdentityResourceId }} + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + STORAGEACCOUNT_LOCATION: ${{ parameters.inputs.storageAccountLocation }} + STORAGEACCOUNT_NAME: ${{ parameters.inputs.storageAccountName }} + + ${{ elseif eq(parameters.resourceType, 'roledefinition') }}: + ROLEDEFINITION_JSON: ${{ parameters.inputs.roleDefinitionJson }} + + ${{ elseif eq(parameters.resourceType, 'managedidentity') }}: + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + MANAGEDIDENTITY_NAME: ${{ parameters.inputs.managedIdentityName }} + MANAGEDIDENTITY_LOCATION: ${{ parameters.inputs.managedIdentityLocation }} + + ${{ elseif eq(parameters.resourceType, 'serviceconnection') }}: + AZURE_DEVOPS_EXT_AZURE_RM_SERVICE_PRINCIPAL_KEY: $(System.AccessToken) + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + SERVICECONNECTION_PRINCIPALID: ${{ parameters.inputs.serviceConnectionPrincipalId }} + SUBSCRIPTION_ID: ${{ parameters.inputs.subscriptionId }} + SUBSCRIPTION_NAME: ${{ parameters.inputs.subscriptionName }} + SERVICECONNECTION_TENANT_ID: ${{ parameters.inputs.tenantId }} + SERVICECONNECTION_NAME: ${{ parameters.inputs.serviceConnectionName }} + + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + + ${{ if eq(parameters.resourceType, 'roleassignments') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + az upgrade --yes + az role assignment create --help + + az role assignment create \ + --role "$ROLE_NAME" \ + --assignee "$MANAGEDIDENTITY_NAME" \ + --resource-group "$RESOURCEGROUP_NAME" \ + --scope "$MI_SCOPE" + + ${{ elseif eq(parameters.resourceType, 'resourcegroups') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az group create \ + --name "$RESOURCEGROUP_NAME" \ + --location "$RESOURCEGROUP_LOCATION" \ + --tags "$BUILDTAG_DEFINITIONID"="$SYSTEM_DEFINITIONID" \ + "$BUILDTAG_CREATEDBYBUILDID"="$BUILD_BUILDID" \ + "$BUILDTAG_CREATEDBYAPPID"="$servicePrincipalId" + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + inlineScript: | + set -e + #[[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + az upgrade -y + + az storage account create \ + --name "$STORAGEACCOUNT_NAME" \ + --location "$STORAGEACCOUNT_LOCATION" \ + --resource-group "$RESOURCEGROUP_NAME" \ + --allow-blob-public-access false \ + --allow-shared-key-access false \ + --tags "$BUILDTAG_DEFINITIONID"="$SYSTEM_DEFINITIONID" \ + "$BUILDTAG_CREATEDBYBUILDID"="$BUILD_BUILDID" \ + "$BUILDTAG_CREATEDBYAPPID"="$servicePrincipalId" + + ${{ elseif eq(parameters.resourceType, 'roledefinition') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az role definition create --role-definition "$ROLEDEFINITION_JSON" + + ${{ elseif eq(parameters.resourceType, 'managedidentity') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az identity create \ + --name "$MANAGEDIDENTITY_NAME" \ + --location "$MANAGEDIDENTITY_LOCATION" \ + --resource-group "$RESOURCEGROUP_NAME" \ + --tags "$ACNCI_BUILDTAG_DEFINITIONID"="$SYSTEM_DEFINITIONID" \ + "$ACNCI_BUILDTAG_CREATEDBYBUILDID"="$BUILD_BUILDID" \ + "$ACNCI_BUILDTAG_CREATEDBYAPPID"="$servicePrincipalId" + + + + - ${{ if parameters.refreshAfterCreation }}: + # Update data list + - template: get-resources.steps.yaml + parameters: + resourceType: ${{ parameters.resourceType }} + serviceConnection: ${{ parameters.serviceConnection }} + inputs: + buildTagDefinitionIdKey: ${{ parameters.inputs.buildTagDefinitionIdKey }} + buildTagCreatedByAppIdKey: ${{ parameters.inputs.buildTagCreatedByAppIdKey }} + ${{ if eq(parameters.resourceType, 'managedidentity') }}: + resourceGroupName: ${{ parameters.inputs.resourceGroupName }} + + ${{ elseif eq(parameters.resourceType, 'roledefinition') }}: + roleName: ${{ parameters.inputs.roleName }} + roleDefinitionFileLocation: ${{ parameters.inputs.roleDefinitionFileLocation }} + subscriptionId: ${{ parameters.inputs.subscriptionId }} + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + resourceGroupName: ${{ parameters.inputs.resourceGroupName }} + + +- ${{ if and(parameters.updateCondition, or( eq(parameters.resourceType, 'roledefinition'), eq(parameters.resourceType, 'storageaccounts') )) }}: + - task: AzureCLI@2 + displayName: "[Update] ARM Details - (${{ parameters.resourceType }})" + condition: ${{ parameters.updateCondition }} + + env: + BUILDTAG_DEFINITIONID: ${{ parameters.inputs.buildTagDefinitionIdKey }} + BUILDTAG_CREATEDBYAPPID: ${{ parameters.inputs.buildTagCreatedByAppIdKey }} + ${{ if eq(parameters.resourceType, 'roledefinition') }}: + ROLEDEFINITION_JSON: ${{ parameters.inputs.roleDefinitionJson }} + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + STORAGEACCOUNT_NAME: ${{ parameters.inputs.storageAccountName }} + + ${{ elseif eq(parameters.resourceType, 'managedidentity') }}: + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + MANAGEDIDENTITY_NAME: ${{ parameters.inputs.managedIdentityName }} + MANAGEDIDENTITY_FEDCRED_NAME: ${{ parameters.inputs.managedIdentityName }}-cred + + + + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + + ${{ if eq(parameters.resourceType, 'roledefinition') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az role definition update --role-definition "$ROLEDEFINITION_JSON" + + ${{ elseif eq(parameters.resourceType, 'roleassignments') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + echo "$ROLEASSIGNMENTS_LIST" | \ + jq -rc \ + --arg RA_NAME "$RA_NAME" \ + --arg RA_PRINCIPAL_ID "$RA_PRINCIPAL_ID" \ + --arg RA_SCOPE "$RA_SCOPE" \ + '. | select(.roleDefinitionName == $RA_NAME && .principalId == $RA_PRINCIPAL_ID && .scope == $RA_SCOPE)' + + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az storage account update \ + --name "$STORAGEACCOUNT_NAME" \ + --resource-group "$RESOURCEGROUP_NAME" \ + --assign-identity + + ${{ elseif eq(parameters.resourceType, 'managedidentity') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + CONDITION="(@Resource[Microsoft.Storage/storageAccounts/blobServices/containers:Name] stringEquals '$STORAGECONTAINER_NAME' AND (!(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read'} AND NOT SubOperationMatches{'Blob.List'}) OR @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/blobs/tags:'$BUILDTAG_BUILDUSER_APPID'<\$key_case_sensitive\$>] StringEquals '$MANAGEDIDENTITY_APPID'))" + az role assignment update \ + --role "$ROLE_NAME" \ + --assignee-object-id "$SERVICEPRINCIPAL_CLIENTID" \ + --assignee-principl-type "ServicePrincipal" \ + --scope "$RESOURCEGROUP_ID" \ + --description "Enable access to build user blobs." \ + --condition "$CONDITION" \ + --condition-version "2.0" + + #FEDCRED_DETAILS=$(az identity federated-credential create \ + #--name "$MANAGEDIDENTITY_FEDCRED_NAME" \ + #--identity-name "$MANAGEDIDENTITY_NAME" \ + #--resource-group "$RESOURCEGROUP_NAME" \ + #--issuer "https://VisualStudio/SPN" \ + #--subject "user_impersonation" \ + #--audience "api://AzureADMyOrg") + #echo $FEDCRED_DETAILS diff --git a/.pipelines/templates/download-storage-artifacts.steps.yaml b/.pipelines/templates/download-storage-artifacts.steps.yaml new file mode 100644 index 0000000000..c4fc900129 --- /dev/null +++ b/.pipelines/templates/download-storage-artifacts.steps.yaml @@ -0,0 +1,32 @@ +parameters: +- name: artifactName + type: string + +- name: targetFolder + type: string + default: $(Build.ArtifactStagingDirectory) + + +steps: + +- task: AzureCLI@2 + displayName: "[Artifacts] Download Blob" + inputs: + azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + mkdir -p "$DOWNLOAD_TARGET/$ARTIFACT_NAME" + az storage blob download-batch \ + --account-name "$SA_NAME" \ + --destination "$DOWNLOAD_TARGET/$ARTIFACT_NAME" \ + --source "$CONTAINER_NAME" \ + --pattern "$BLOB_PATH/$ARTIFACT_NAME/*" + env: + ARTIFACT_NAME: ${{ parameters.artifactName }} + DOWNLOAD_TARGET: ${{ parameters.targetFolder }} + diff --git a/.pipelines/templates/get-resources.steps.yaml b/.pipelines/templates/get-resources.steps.yaml new file mode 100644 index 0000000000..2a463c7d45 --- /dev/null +++ b/.pipelines/templates/get-resources.steps.yaml @@ -0,0 +1,202 @@ + +parameters: +- name: serviceConnection + type: string + +- name: resourceType + type: string + values: + - roleassignments + - resourcegroups + - storageaccounts + - roledefinition + - managedidentity + - serviceconnection + +- name: inputs + type: object + default: {} + +- name: outputVariableName + type: string + default: 'OUT_RESULT' + +steps: +- task: AzureCLI@2 + displayName: "[Get] ARM Details - (${{ parameters.resourceType }})" + + env: + VAR_NAME: ${{ parameters.outputVariableName }} + BUILDTAG_DEFINITIONID: ${{ parameters.inputs.buildTagDefinitionIdKey }} + BUILDTAG_CREATEDBYAPPID: ${{ parameters.inputs.buildTagCreatedByAppIdKey }} + ${{ if eq(parameters.resourceType, 'roledefinition') }}: + SUBSCRIPTION_ID: ${{ parameters.inputs.subscriptionId }} + ROLE_NAME: ${{ parameters.inputs.roleName }} + ROLEDEFINITION_FILEPATH: ${{ parameters.inputs.roleDefinitionFileLocation }} + + ${{ elseif eq(parameters.resourceType, 'roleassignments') }}: + #MANAGEDIDENTITY_OBJECTID: ${{ parameters.inputs.managedIdentityObjectId }} + #RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + #ROLE_NAME: ${{ parameters.inputs.roleName }} + MANAGEDIDENTITY_NAME: ${{ parameters.inputs.managedIdentityObjectId }} + MANAGEDIDENTITY_SCOPE: ${{ parameters.inputs.scope }} + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + ROLE_NAME: ${{ parameters.inputs.roleName }} + + ${{ elseif eq(parameters.resourceType, 'resourcegroups') }}: + # Allows use of az devops commands + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + ${{ if parameters.inputs.storageAccountName }}: + STORAGEACCOUNT_NAME: ${{ parameters.inputs.storageAccountName }} + + ${{ elseif eq(parameters.resourceType, 'managedidentity') }}: + RESOURCEGROUP_NAME: ${{ parameters.inputs.resourceGroupName }} + + ${{ elseif eq(parameters.resourceType, 'serviceconnection') }}: + # Allows use of az devops commands + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + SERVICECONNECTION_ID: ${{ parameters.inputs.serviceConnectionId }} + + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + + ${{ if eq(parameters.resourceType, 'roledefinition') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + DEF_LIST=$(az role definition list \ + --name "$ROLE_NAME" \ + --custom-role-only \ + -ojson | jq -rc '.') + DEF=$(echo "$DEF_LIST" | jq -rc .[0]) + DEFS_FOUND=$(echo "$DEF_LIST" | jq length) + + if [[ -z $DEF_LIST ]] || \ + [[ $DEFS_FOUND == 'null' ]] || \ + (( $DEFS_FOUND < 1 )); then + + DEF=$(cat "$ROLEDEFINITION_FILEPATH" | \ + jq -rc \ + --arg RESOURCEID "/subscriptions/$SUBSCRIPTION_ID" \ + '.assignableScopes[] = $RESOURCEID') + fi + + echo >&2 "##vso[task.setvariable variable=${VAR_NAME};]$DEF" + echo >&2 "##vso[task.setvariable variable=${VAR_NAME}_LENGTH;]$DEFS_FOUND" + + + ${{ elseif eq(parameters.resourceType, 'roleassignments') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + MI_ROLE_DATA=$(az role assignment list \ + --role "$ROLE_NAME" \ + --assignee "$MANAGEDIDENTITY_NAME" \ + --scope "$MI_SCOPE" \ + --output json | jq -rc '.') + echo $MI_ROLE_DATA + #R_QUERY="[? name == '$MANAGEDIDENTITY_OBJECTID' ]" + #MI_ROLE_DATA=$(az role assignment list \ + # --role "$ROLE_NAME" \ + # --resource-group "$RESOURCEGROUP_NAME" \ + # --query "$R_QUERY" \ + # --output json | jq -rc '.') + MI_ROLE_DATA_LENGTH=$(echo "$MI_ROLE_DATA" | jq length) + + echo >&2 "##vso[task.setvariable variable=${VAR_NAME};]$MI_ROLE_DATA" + echo >&2 "##vso[task.setvariable variable=${VAR_NAME}_LENGTH;]$MI_ROLE_DATA_LENGTH" + + + ${{ elseif eq(parameters.resourceType, 'resourcegroups') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + # Get Pipeline Build Subscriptions. + R_QUERY="[? tags.\""$BUILDTAG_DEFINITIONID"\" && tags.\""$BUILDTAG_CREATEDBYAPPID"\"]" + INFRA_RG_LIST=$(az group list --query "$R_QUERY" \ + --tag "$BUILDTAG_DEFINITIONID"="$SYSTEM_DEFINITIONID" \ + --tag "$BUILDTAG_CREATEDBYAPPID"="$servicePrincipalId" -o json | jq -rc '.') + INFRA_RG_LENGTH=$(echo "$INFRA_RG_LIST" | jq length) + + echo >&2 "##vso[task.setvariable variable=${VAR_NAME};]$INFRA_RG_LIST" + echo >&2 "##vso[task.setvariable variable=${VAR_NAME}_LENGTH;]$INFRA_RG_LENGTH" + + + ${{ elseif eq(parameters.resourceType, 'storageaccounts') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + # Query for Test Storage Accounts + R_QUERY="[? tags.\""$BUILDTAG_DEFINITIONID"\" && tags.\""$BUILDTAG_CREATEDBYAPPID"\"]" + # JSON of Returned Results + R_LIST=$(az storage account list \ + --resource-group "$RESOURCEGROUP_NAME" \ + --query "$R_QUERY" | \ + jq -rc \ + --arg BUILDTAG_CREATEDBYAPPID "$BUILDTAG_CREATEDBYAPPID" \ + --arg APPID "$servicePrincipalId" \ + --arg BUILDTAG_DEFINITIONID "$BUILDTAG_DEFINITIONID" \ + --arg DEFINITIONID "$SYSTEM_DEFINITIONID" \ + '[ .[] | select(.tags[$BUILDTAG_DEFINITIONID] == $DEFINITIONID ) | select( .tags[$BUILDTAG_CREATEDBYAPPID] == $APPID) ]') + + if [[ -n "$STORAGEACCOUNT_NAME" ]]; then + R_LIST=$(echo "$R_LIST" | | jq -rc --arg STORAGEACCOUNT_NAME "$STORAGEACCOUNT_NAME" '[.[] | select(.name == $STORAGEACCOUNT_NAME)]') + fi + # Length of resource list + R_LIST_LENGTH=$(echo "$R_LIST" | jq length) + + # Export the available storage account list.. + # uses custom variable naming if specified. + echo >&2 "##vso[task.setvariable variable=${VAR_NAME};]$R_LIST" + echo >&2 "##vso[task.setvariable variable=${VAR_NAME}_LENGTH;]$R_LIST_LENGTH" + + + ${{ elseif eq(parameters.resourceType, 'roleassignments') }}: + inlineScript: | + set -e + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + RA_LIST=$(az role assignment list \ + --assignee "$ASSIGNEE_ID" \ + --include-groups \ + --include-inherited) + + RA_LIST_LENGTH=$(echo "$RA_LIST" | jq length) + + echo >&2 "##vso[task.setvariable variable=${VAR_NAME};]$RA_LIST" + echo >&2 "##vso[task.setvariable variable=${VAR_NAME}_LENGTH;]$RA_LIST_LENGTH" + + ${{ elseif eq(parameters.resourceType, 'serviceconnection') }}: + inlineScript: | + set -eu + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + az devops service-endpoint show \ + --id "$SERVICECONNECTION_ID" \ + --org "https://dev.azure.com/msazure/" \ + --project One \ + -ojson --debug + # JSON of Returned Results + R_LIST=$(az devops service-endpoint show \ + --id "$SERVICECONNECTION_ID" \ + --org "https://dev.azure.com/msazure/" \ + --project One \ + -ojson | \ + jq -rc '[.]') + DEFS_FOUND=$(echo "$R_LIST" | jq length) + echo "$R_LIST" | jq '.' + + # Export the available storage account list.. + # uses custom variable naming if specified. + echo >&2 "##vso[task.setvariable variable=${VAR_NAME};]$R_LIST" + echo >&2 "##vso[task.setvariable variable=${VAR_NAME}_LENGTH;]$DEFS_FOUND" diff --git a/.pipelines/templates/get-storage-accounts.steps.yaml b/.pipelines/templates/get-storage-accounts.steps.yaml new file mode 100644 index 0000000000..2189bd11ba --- /dev/null +++ b/.pipelines/templates/get-storage-accounts.steps.yaml @@ -0,0 +1,61 @@ +parameters: + +- name: STORAGE_ACCOUNT_SERVICE_CONNECTION + type: string + +- name: condition + type: object + default: true + +- name: resourceGroupName + type: string + +- name: varNameStorageAccountList + type: string + default: None + +steps: +## Storage Account Resource Query +- task: AzureCLI@2 + displayName: "Get Storage Account Resources" + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.STORAGE_ACCOUNT_SERVICE_CONNECTION }} + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + set -eu + [[ -n $SYSTEM_DEBUG ]] && [[ $SYSTEM_DEBUG =~ $IS_TRUE ]] && set -x || set +x + + # Query for Test Storage Accounts + R_QUERY="[? type == 'Microsoft.Storage/storageAccounts' && resourceGroup == '$RG_NAME']" + az resource list \ + --query "$R_QUERY" \ + --tag "$ACNCI_BUILDTAG_DEFINITIONID"="$SYSTEM_DEFINITIONID" \ + --tag "$ACNCI_BUILDTAG_CREATEDBYAPPID"="$servicePrincipalId" -o json + echo $R_QUERY + + # JSON of Returned Results + R_LIST=$(az resource list \ + --query "$R_QUERY" \ + --tag "$ACNCI_BUILDTAG_DEFINITIONID"="$SYSTEM_DEFINITIONID" \ + --tag "$ACNCI_BUILDTAG_CREATEDBYAPPID"="$servicePrincipalId" -o json | jq -rc '.') + echo $R_LIST + # Length of resource list + R_LIST_LENGTH=$(echo "$R_LIST" | jq length) + echo $R_LIST_LENGTH + + # Export the available storage account list.. + # uses custom variable naming if specified. + if [[ -n $VARNAME_SA_LIST ]] && \ + ! [[ $VARNAME_SA_LIST =~ ^[N|n][O|o][N|n][E|e]$ ]]; then + echo >&2 "##vso[task.setvariable variable=${VARNAME_SA_LIST}]${R_LIST}" + echo >&2 "##vso[task.setvariable variable=${VARNAME_SA_LIST}_LENGTH]${R_LIST_LENGTH}" + else + echo >&2 "##vso[task.setvariable variable=SA_LIST;]$R_LIST" + echo >&2 "##vso[task.setvariable variable=SA_LIST_LENGTH;]$R_LIST_LENGTH" + fi + env: + VARNAME_SA_LIST: '${{ parameters.varNameStorageAccountList }}' + RG_NAME: '${{ parameters.resourceGroupName }}' diff --git a/.pipelines/templates/mi-build-role.json b/.pipelines/templates/mi-build-role.json new file mode 100644 index 0000000000..30a451ea72 --- /dev/null +++ b/.pipelines/templates/mi-build-role.json @@ -0,0 +1,49 @@ +{ + "Name": "ACN CI/CD Build Environment Owner", + "roleName": "ACN CI/CD Build Environment Owner", + "Type": "CustomRole", + "roleType": "CustomRole", + "isCustom": true, + "description": "A custom role given to managed identities created by the CI build process. This role should try to focus on owning the resources already in its scope; rather than creating new ones. Actions should be added as necessary - and not in anticipation.", + "actions": [ + "Microsoft.Storage/storageAccounts/read", + + "Microsoft.Storage/storageAccounts/blobServices/generateUserDelegationKey/action", + "Microsoft.Storage/storageAccounts/blobServices/containers/read", + "Microsoft.Storage/storageAccounts/blobServices/containers/write", + + "Microsoft.Storage/storageAccounts/tableServices/tables/read", + "Microsoft.Storage/storageAccounts/tableServices/tables/write", + + "Microsoft.ContainerRegistry/registries/pull/read", + "Microsoft.ContainerRegistry/registries/push/write" + ], + "notActions": [], + "dataActions": [ + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/delete", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/filter/action", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/move/action", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/add/action", + + "Microsoft.Storage/storageAccounts/tableServices/tables/entities/read", + "Microsoft.Storage/storageAccounts/tableServices/tables/entities/write", + "Microsoft.Storage/storageAccounts/tableServices/tables/entities/delete", + "Microsoft.Storage/storageAccounts/tableServices/tables/entities/add/action", + "Microsoft.Storage/storageAccounts/tableServices/tables/entities/update/action", + + "Microsoft.Storage/storageAccounts/fileServices/readFileBackupSemantics/action", + "Microsoft.Storage/storageAccounts/fileServices/writeFileBackupSemantics/action", + "Microsoft.Storage/storageAccounts/fileServices/fileshares/files/read", + "Microsoft.Storage/storageAccounts/fileServices/fileshares/files/write", + "Microsoft.Storage/storageAccounts/fileServices/fileshares/files/delete", + "Microsoft.Storage/storageAccounts/fileServices/fileshares/files/modifypermissions/action", + + "Microsoft.Storage/storageAccounts/queueServices/queues/messages/*" + ], + "notDataActions": [], + "assignableScopes": [ + "$RESOURCEID" + ] +}