Speed up mac builds

build/azure-pipelines/darwin/product-build-darwin-sign.yml

@@ -0,0 +1,129 @@
+steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: "12.18.3"
+
+  - task: AzureKeyVault@1
+    displayName: "Azure Key Vault: Get Secrets"
+    inputs:
+      azureSubscription: "vscode-builds-subscription"
+      KeyVaultName: vscode
+
+  - script: |
+      set -e
+      cat << EOF > ~/.netrc
+      machine github.com
+      login vscode
+      password $(github-distro-mixin-password)
+      EOF
+
+      git config user.email "vscode@microsoft.com"
+      git config user.name "VSCode"
+    displayName: Prepare tooling
+
+  - script: |
+      set -e
+      git pull --no-rebase https://github.com/$(VSCODE_MIXIN_REPO).git $(node -p "require('./package.json').distro")
+    displayName: Merge distro
+
+  - script: |
+      pushd build \
+      && yarn \
+      && npm install -g typescript \
+      && tsc azure-pipelines/common/createAsset.ts \
+      && popd
+    displayName: Restore modules for just build folder and compile it
+
+  - download: current
+    artifact: vscode-darwin-$(VSCODE_ARCH)
+    displayName: Download $(VSCODE_ARCH) artifact
+
+  - script: |
+      set -e
+      unzip $(Pipeline.Workspace)/vscode-darwin-$(VSCODE_ARCH)/VSCode-darwin-$(VSCODE_ARCH).zip -d $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      mv $(Pipeline.Workspace)/vscode-darwin-$(VSCODE_ARCH)/VSCode-darwin-$(VSCODE_ARCH).zip $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH).zip
+    displayName: Unzip & move
+
+  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+    inputs:
+      ConnectedServiceName: "ESRP CodeSign"
+      FolderPath: "$(agent.builddirectory)"
+      Pattern: "VSCode-darwin-$(VSCODE_ARCH).zip"
+      signConfigType: inlineSignParams
+      inlineOperation: |
+        [
+          {
+            "keyCode": "CP-401337-Apple",
+            "operationSetCode": "MacAppDeveloperSign",
+            "parameters": [
+              {
+                "parameterName": "Hardening",
+                "parameterValue": "--options=runtime"
+              }
+            ],
+            "toolName": "sign",
+            "toolVersion": "1.0"
+          }
+        ]
+      SessionTimeout: 60
+    displayName: Codesign
+
+  - script: |
+      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      BUNDLE_IDENTIFIER=$(node -p "require(\"$APP_ROOT/$APP_NAME/Contents/Resources/app/product.json\").darwinBundleIdentifier")
+      echo "##vso[task.setvariable variable=BundleIdentifier]$BUNDLE_IDENTIFIER"
+    displayName: Export bundle identifier
+
+  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+    inputs:
+      ConnectedServiceName: "ESRP CodeSign"
+      FolderPath: "$(agent.builddirectory)"
+      Pattern: "VSCode-darwin-$(VSCODE_ARCH).zip"
+      signConfigType: inlineSignParams
+      inlineOperation: |
+        [
+          {
+            "keyCode": "CP-401337-Apple",
+            "operationSetCode": "MacAppNotarize",
+            "parameters": [
+              {
+                "parameterName": "BundleId",
+                "parameterValue": "$(BundleIdentifier)"
+              }
+            ],
+            "toolName": "sign",
+            "toolVersion": "1.0"
+          }
+        ]
+      SessionTimeout: 60
+    displayName: Notarization
+
+  - script: |
+      set -e
+      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      "$APP_ROOT/$APP_NAME/Contents/Resources/app/bin/code" --export-default-configuration=.build
+    displayName: Verify start after signing (export configuration)
+    condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - script: |
+      set -e
+
+      # For legacy purposes, arch for x64 is just 'darwin'
+      case $VSCODE_ARCH in
+        x64) ASSET_ID="darwin" ;;
+        arm64) ASSET_ID="darwin-arm64" ;;
+        universal) ASSET_ID="darwin-universal" ;;
+      esac
+
+      VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
+      AZURE_DOCUMENTDB_MASTERKEY="$(builds-docdb-key-readwrite)" \
+      AZURE_STORAGE_ACCESS_KEY="$(ticino-storage-key)" \
+      AZURE_STORAGE_ACCESS_KEY_2="$(vscode-storage-key)" \
+      node build/azure-pipelines/common/createAsset.js \
+        "$ASSET_ID" \
+        archive \
+        "VSCode-$ASSET_ID.zip" \
+        ../VSCode-darwin-$(VSCODE_ARCH).zip
+    displayName: Publish Clients

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -22,6 +22,8 @@ steps:
     displayName: Extract compilation output
     condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'universal'))
 
+  # Set up the credentials to retrieve distro repo and setup git persona
+  # to create a merge commit for when we merge distro into oss
   - script: |
       set -e
       cat << EOF > ~/.netrc
@@ -119,16 +121,7 @@ steps:
     displayName: Rebuild native modules for ARM64
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'arm64'))
 
-  - download: current
-    artifact: vscode-darwin-x64
-    displayName: Download x64 artifact
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
-
-  - download: current
-    artifact: vscode-darwin-arm64
-    displayName: Download arm64 artifact
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
-
+  # This script brings in the right resources (images, icons, etc) based on the quality (insiders, stable, exploration)
   - script: |
       set -e
       node build/azure-pipelines/mixin
@@ -138,7 +131,7 @@ steps:
       set -e
       VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
         yarn gulp vscode-darwin-$(VSCODE_ARCH)-min-ci
-    displayName: Build
+    displayName: Build client
     condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'universal'))
 
   - script: |
@@ -150,21 +143,36 @@ steps:
     displayName: Build Server
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'))
 
-  - script: |
-      set -e
-      unzip $(Pipeline.Workspace)/vscode-darwin-x64/VSCode-darwin-x64.zip -d $(agent.builddirectory)/vscode-x64
-      unzip $(Pipeline.Workspace)/vscode-darwin-arm64/VSCode-darwin-arm64.zip -d $(agent.builddirectory)/vscode-arm64
-      DEBUG=* node build/darwin/create-universal-app.js
-    displayName: Create Universal App
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
-
   - script: |
       set -e
       VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
         yarn npm-run-all -lp "electron $(VSCODE_ARCH)" "playwright-install"
     displayName: Download Electron and Playwright
     condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'universal'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
+  - download: current
+    artifact: vscode-darwin-x64
+    displayName: Download x64 artifact
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
+
+  - download: current
+    artifact: vscode-darwin-arm64
+    displayName: Download arm64 artifact
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
+
+  - script: |
+      set -e
+      cp $(Pipeline.Workspace)/vscode-darwin-x64/VSCode-darwin-x64.zip $(agent.builddirectory)/VSCode-darwin-x64.zip
+      cp $(Pipeline.Workspace)/vscode-darwin-arm64/VSCode-darwin-arm64.zip $(agent.builddirectory)/VSCode-darwin-arm64.zip
+      unzip $(agent.builddirectory)/VSCode-darwin-x64.zip -d $(agent.builddirectory)/VSCode-darwin-x64
+      unzip $(agent.builddirectory)/VSCode-darwin-arm64.zip -d $(agent.builddirectory)/VSCode-darwin-arm64
+      DEBUG=* node build/darwin/create-universal-app.js
+    displayName: Create Universal App
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
+
+  # Setting hardened entitlements is a requirement for:
+  # * Apple notarization
+  # * Running tests on Big Sur (because Big Sur has additional security precautions)
   - script: |
       set -e
       security create-keychain -p pwd $(agent.tempdirectory)/buildagent.keychain
@@ -173,9 +181,8 @@ steps:
       echo "$(macos-developer-certificate)" | base64 -D > $(agent.tempdirectory)/cert.p12
       security import $(agent.tempdirectory)/cert.p12 -k $(agent.tempdirectory)/buildagent.keychain -P "$(macos-developer-certificate-key)" -T /usr/bin/codesign
       security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k pwd $(agent.tempdirectory)/buildagent.keychain
-      VSCODE_ARCH="$(VSCODE_ARCH)" DEBUG=electron-osx-sign* node build/darwin/sign.js
+      VSCODE_ARCH=$(VSCODE_ARCH) DEBUG=electron-osx-sign* node build/darwin/sign.js
     displayName: Set Hardened Entitlements
-    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
 
   - script: |
       set -e
@@ -284,91 +291,19 @@ steps:
     displayName: Archive build
     condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
 
-  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
-    inputs:
-      ConnectedServiceName: "ESRP CodeSign"
-      FolderPath: "$(agent.builddirectory)"
-      Pattern: "VSCode-darwin-$(VSCODE_ARCH).zip"
-      signConfigType: inlineSignParams
-      inlineOperation: |
-        [
-          {
-            "keyCode": "CP-401337-Apple",
-            "operationSetCode": "MacAppDeveloperSign",
-            "parameters": [
-              {
-                "parameterName": "Hardening",
-                "parameterValue": "--options=runtime"
-              }
-            ],
-            "toolName": "sign",
-            "toolVersion": "1.0"
-          }
-        ]
-      SessionTimeout: 60
-    displayName: Codesign
-    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
-
-  - script: |
-      zip -d $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH).zip "*.pkg"
-    displayName: Clean
-    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
-
-  - script: |
-      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      APP_NAME="`ls $APP_ROOT | head -n 1`"
-      BUNDLE_IDENTIFIER=$(node -p "require(\"$APP_ROOT/$APP_NAME/Contents/Resources/app/product.json\").darwinBundleIdentifier")
-      echo "##vso[task.setvariable variable=BundleIdentifier]$BUNDLE_IDENTIFIER"
-    displayName: Export bundle identifier
-    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
-
-  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
-    inputs:
-      ConnectedServiceName: "ESRP CodeSign"
-      FolderPath: "$(agent.builddirectory)"
-      Pattern: "VSCode-darwin-$(VSCODE_ARCH).zip"
-      signConfigType: inlineSignParams
-      inlineOperation: |
-        [
-          {
-            "keyCode": "CP-401337-Apple",
-            "operationSetCode": "MacAppNotarize",
-            "parameters": [
-              {
-                "parameterName": "BundleId",
-                "parameterValue": "$(BundleIdentifier)"
-              }
-            ],
-            "toolName": "sign",
-            "toolVersion": "1.0"
-          }
-        ]
-      SessionTimeout: 60
-    displayName: Notarization
-    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
-
-  - script: |
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      APP_NAME="`ls $APP_ROOT | head -n 1`"
-      "$APP_ROOT/$APP_NAME/Contents/Resources/app/bin/code" --export-default-configuration=.build
-    displayName: Verify start after signing (export configuration)
-    condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'arm64'), ne(variables['VSCODE_PUBLISH'], 'false'))
-
   - script: |
       set -e
       VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
       AZURE_DOCUMENTDB_MASTERKEY="$(builds-docdb-key-readwrite)" \
       AZURE_STORAGE_ACCESS_KEY="$(ticino-storage-key)" \
       AZURE_STORAGE_ACCESS_KEY_2="$(vscode-storage-key)" \
-      VSCODE_ARCH="$(VSCODE_ARCH)" \
-      ./build/azure-pipelines/darwin/publish.sh
-    displayName: Publish
-    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
+      VSCODE_ARCH="$(VSCODE_ARCH)" ./build/azure-pipelines/darwin/publish-server.sh
+    displayName: Publish Servers
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), ne(variables['VSCODE_PUBLISH'], 'false'))
 
   - publish: $(Agent.BuildDirectory)/VSCode-darwin-$(VSCODE_ARCH).zip
     artifact: vscode-darwin-$(VSCODE_ARCH)
-    displayName: Publish archive
+    displayName: Publish client archive
     condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
 
   - publish: $(Agent.BuildDirectory)/vscode-server-darwin.zip
@@ -386,5 +321,5 @@ steps:
       VSCODE_ARCH="$(VSCODE_ARCH)" \
       yarn gulp upload-vscode-configuration
     displayName: Upload configuration (for Bing settings search)
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), ne(variables['VSCODE_PUBLISH'], 'false'))
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'))
     continueOnError: true

build/azure-pipelines/darwin/publish.sh → build/azure-pipelines/darwin/publish-server.sh

@@ -1,20 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-# Publish DEB
-case $VSCODE_ARCH in
-	x64) ASSET_ID="darwin" ;;
-	arm64) ASSET_ID="darwin-arm64" ;;
-	universal) ASSET_ID="darwin-universal" ;;
-esac
-
-# publish the build
-node build/azure-pipelines/common/createAsset.js \
-	"$ASSET_ID" \
-	archive \
-	"VSCode-$ASSET_ID.zip" \
-	../VSCode-darwin-$VSCODE_ARCH.zip
-
 if [ "$VSCODE_ARCH" == "x64" ]; then
 	# package Remote Extension Host
 	pushd .. && mv vscode-reh-darwin vscode-server-darwin && zip -Xry vscode-server-darwin.zip vscode-server-darwin && popd

build/azure-pipelines/product-build.yml

@@ -254,6 +254,15 @@ stages:
               VSCODE_ARCH: x64
             steps:
               - template: darwin/product-build-darwin.yml
+          - ${{ if ne(variables['VSCODE_PUBLISH'], 'false') }}:
+            - job: macOSSign
+              dependsOn:
+                - macOS
+              timeoutInMinutes: 90
+              variables:
+                VSCODE_ARCH: x64
+              steps:
+                - template: darwin/product-build-darwin-sign.yml
 
         - ${{ if and(eq(variables['VSCODE_CIBUILD'], false), eq(parameters.VSCODE_BUILD_MACOS_ARM64, true)) }}:
           - job: macOSARM64
@@ -262,6 +271,15 @@ stages:
               VSCODE_ARCH: arm64
             steps:
               - template: darwin/product-build-darwin.yml
+        - ${{ if ne(variables['VSCODE_PUBLISH'], 'false') }}:
+            - job: macOSARM64Sign
+              dependsOn:
+                - macOSARM64
+              timeoutInMinutes: 90
+              variables:
+                VSCODE_ARCH: arm64
+              steps:
+                - template: darwin/product-build-darwin-sign.yml
 
         - ${{ if eq(variables['VSCODE_BUILD_MACOS_UNIVERSAL'], true) }}:
           - job: macOSUniversal
@@ -273,6 +291,15 @@ stages:
               VSCODE_ARCH: universal
             steps:
               - template: darwin/product-build-darwin.yml
+          - ${{ if ne(variables['VSCODE_PUBLISH'], 'false') }}:
+            - job: macOSUniversalSign
+              dependsOn:
+                - macOSUniversal
+              timeoutInMinutes: 90
+              variables:
+                VSCODE_ARCH: universal
+              steps:
+                - template: darwin/product-build-darwin-sign.yml
 
   - ${{ if and(eq(variables['VSCODE_PUBLISH'], true), eq(parameters.VSCODE_COMPILE_ONLY, false)) }}:
     - stage: Mooncake