Initial move to a Publish stage (#122886)

* first attempt

actually run

ok run it for real

actually work

parameter syntax

monaco

logging

env var is always string

use variables

use global params

try idea to stop looping

move variables

wrap in quotes

rename all the artifacts

try downloading

up timeout

misc

underscore

add auth

log everything we need to

typo

typo

publish linux client archives too

renamed all that needed to be renamed

better error handling

raname windows assets

first attempt at letting new code handle releasing with macOS

move to $env

remove gate

include updated createAsset.js

onboard windows client

include js

add linux client

add built js

move over all publishing

update distro

use branch name?

processed artifacts and fix win32 server platforms

publish what artifacts were published

use download and more more to powershell

put it all in exec

actually in directory

fix path again

create file to be uploaded

move to all PowerShell

rename to publish and clean up code

update createAsset

use distro 2

more commented out code and add displayNames

address feedback

* rev distro

* rev distro

build/azure-pipelines/common/createAsset.js

@@ -10,10 +10,95 @@ const azure = require("azure-storage");
 const mime = require("mime");
 const cosmos_1 = require("@azure/cosmos");
 const retry_1 = require("./retry");
-if (process.argv.length !== 6) {
-    console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
+if (process.argv.length !== 8) {
+    console.error('Usage: node createAsset.js PRODUCT OS ARCH TYPE NAME FILE');
     process.exit(-1);
 }
+// Contains all of the logic for mapping details to our actual product names in CosmosDB
+function getPlatform(product, os, arch, type) {
+    switch (os) {
+        case 'win32':
+            switch (product) {
+                case 'client':
+                    const asset = arch === 'ia32' ? 'win32' : `win32-${arch}`;
+                    switch (type) {
+                        case 'archive':
+                            return `${asset}-archive`;
+                        case 'setup':
+                            return asset;
+                        case 'user-setup':
+                            return `${asset}-user`;
+                        default:
+                            throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+                    }
+                case 'server':
+                    if (arch === 'arm64') {
+                        throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+                    }
+                    return arch === 'ia32' ? 'server-win32' : `server-win32-${arch}`;
+                case 'web':
+                    if (arch === 'arm64') {
+                        throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+                    }
+                    return arch === 'ia32' ? 'server-win32-web' : `server-win32-${arch}-web`;
+                default:
+                    throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+            }
+        case 'linux':
+            switch (type) {
+                case 'snap':
+                    return `linux-snap-${arch}`;
+                case 'archive-unsigned':
+                    switch (product) {
+                        case 'client':
+                            return `linux-${arch}`;
+                        case 'server':
+                            return `server-linux-${arch}`;
+                        case 'web':
+                            return arch === 'standalone' ? 'web-standalone' : `server-linux-${arch}-web`;
+                        default:
+                            throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+                    }
+                case 'deb-package':
+                    return `linux-deb-${arch}`;
+                case 'rpm-package':
+                    return `linux-rpm-${arch}`;
+                default:
+                    throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+            }
+        case 'darwin':
+            switch (product) {
+                case 'client':
+                    if (arch === 'x64') {
+                        return 'darwin';
+                    }
+                    return `darwin-${arch}`;
+                case 'server':
+                    return 'server-darwin';
+                case 'web':
+                    if (arch !== 'x64') {
+                        throw `What should the platform be?: ${product} ${os} ${arch} ${type}`;
+                    }
+                    return 'server-darwin-web';
+                default:
+                    throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+            }
+        default:
+            throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+    }
+}
+// Contains all of the logic for mapping types to our actual types in CosmosDB
+function getRealType(type) {
+    switch (type) {
+        case 'user-setup':
+            return 'setup';
+        case 'deb-package':
+        case 'rpm-package':
+            return 'package';
+        default:
+            return type;
+    }
+}
 function hashStream(hashName, stream) {
     return new Promise((c, e) => {
         const shasum = crypto.createHash(hashName);
@@ -45,7 +130,10 @@ function getEnv(name) {
     return result;
 }
 async function main() {
-    const [, , platform, type, fileName, filePath] = process.argv;
+    const [, , product, os, arch, unprocessedType, fileName, filePath] = process.argv;
+    // getPlatform needs the unprocessedType
+    const platform = getPlatform(product, os, arch, unprocessedType);
+    const type = getRealType(unprocessedType);
     const quality = getEnv('VSCODE_QUALITY');
     const commit = getEnv('BUILD_SOURCEVERSION');
     console.log('Creating asset...');
@@ -83,7 +171,7 @@ async function main() {
     console.log('Asset:', JSON.stringify(asset, null, '  '));
     const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
     const scripts = client.database('builds').container(quality).scripts;
-    await retry_1.retry(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
+    await (0, retry_1.retry)(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
 }
 main().then(() => {
     console.log('Asset successfully created');

build/azure-pipelines/common/createAsset.ts

@@ -24,11 +24,98 @@ interface Asset {
 	supportsFastUpdate?: boolean;
 }
 
-if (process.argv.length !== 6) {
-	console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
+if (process.argv.length !== 8) {
+	console.error('Usage: node createAsset.js PRODUCT OS ARCH TYPE NAME FILE');
 	process.exit(-1);
 }
 
+// Contains all of the logic for mapping details to our actual product names in CosmosDB
+function getPlatform(product: string, os: string, arch: string, type: string): string {
+	switch (os) {
+		case 'win32':
+			switch (product) {
+				case 'client':
+					const asset = arch === 'ia32' ? 'win32' : `win32-${arch}`;
+					switch (type) {
+						case 'archive':
+							return `${asset}-archive`;
+						case 'setup':
+							return asset;
+						case 'user-setup':
+							return `${asset}-user`;
+						default:
+							throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+					}
+				case 'server':
+					if (arch === 'arm64') {
+						throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+					}
+					return arch === 'ia32' ? 'server-win32' : `server-win32-${arch}`;
+				case 'web':
+					if (arch === 'arm64') {
+						throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+					}
+					return arch === 'ia32' ? 'server-win32-web' : `server-win32-${arch}-web`;
+				default:
+					throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+			}
+		case 'linux':
+			switch (type) {
+				case 'snap':
+					return `linux-snap-${arch}`;
+				case 'archive-unsigned':
+					switch (product) {
+						case 'client':
+							return `linux-${arch}`;
+						case 'server':
+							return `server-linux-${arch}`;
+						case 'web':
+							return arch === 'standalone' ? 'web-standalone' : `server-linux-${arch}-web`;
+						default:
+							throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+					}
+				case 'deb-package':
+					return `linux-deb-${arch}`;
+				case 'rpm-package':
+					return `linux-rpm-${arch}`;
+				default:
+					throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+			}
+		case 'darwin':
+			switch (product) {
+				case 'client':
+					if (arch === 'x64') {
+						return 'darwin';
+					}
+					return `darwin-${arch}`;
+				case 'server':
+					return 'server-darwin';
+				case 'web':
+					if (arch !== 'x64') {
+						throw `What should the platform be?: ${product} ${os} ${arch} ${type}`;
+					}
+					return 'server-darwin-web';
+				default:
+					throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+			}
+		default:
+			throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
+	}
+}
+
+// Contains all of the logic for mapping types to our actual types in CosmosDB
+function getRealType(type: string) {
+	switch (type) {
+		case 'user-setup':
+			return 'setup';
+		case 'deb-package':
+		case 'rpm-package':
+			return 'package';
+		default:
+			return type;
+	}
+}
+
 function hashStream(hashName: string, stream: Readable): Promise<string> {
 	return new Promise<string>((c, e) => {
 		const shasum = crypto.createHash(hashName);
@@ -68,7 +155,10 @@ function getEnv(name: string): string {
 }
 
 async function main(): Promise<void> {
-	const [, , platform, type, fileName, filePath] = process.argv;
+	const [, , product, os, arch, unprocessedType, fileName, filePath] = process.argv;
+	// getPlatform needs the unprocessedType
+	const platform = getPlatform(product, os, arch, unprocessedType);
+	const type = getRealType(unprocessedType);
 	const quality = getEnv('VSCODE_QUALITY');
 	const commit = getEnv('BUILD_SOURCEVERSION');
 

build/azure-pipelines/darwin/product-build-darwin-sign.yml

@@ -35,13 +35,13 @@ steps:
     displayName: Restore modules for just build folder and compile it
 
   - download: current
-    artifact: vscode-darwin-$(VSCODE_ARCH)
+    artifact: unsigned_vscode_client_darwin_$(VSCODE_ARCH)_archive
     displayName: Download $(VSCODE_ARCH) artifact
 
   - script: |
       set -e
-      unzip $(Pipeline.Workspace)/vscode-darwin-$(VSCODE_ARCH)/VSCode-darwin-$(VSCODE_ARCH).zip -d $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      mv $(Pipeline.Workspace)/vscode-darwin-$(VSCODE_ARCH)/VSCode-darwin-$(VSCODE_ARCH).zip $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH).zip
+      unzip $(Pipeline.Workspace)/unsigned_vscode_client_darwin_$(VSCODE_ARCH)_archive/VSCode-darwin-$(VSCODE_ARCH).zip -d $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      mv $(Pipeline.Workspace)/unsigned_vscode_client_darwin_$(VSCODE_ARCH)_archive/VSCode-darwin-$(VSCODE_ARCH).zip $(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH).zip
     displayName: Unzip & move
 
   - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
@@ -108,22 +108,18 @@ steps:
     condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'arm64'))
 
   - script: |
-      set -e
-
       # For legacy purposes, arch for x64 is just 'darwin'
       case $VSCODE_ARCH in
         x64) ASSET_ID="darwin" ;;
         arm64) ASSET_ID="darwin-arm64" ;;
         universal) ASSET_ID="darwin-universal" ;;
       esac
+      echo "##vso[task.setvariable variable=ASSET_ID]$ASSET_ID"
+    displayName: Set asset id variable
+
+  - script: mv $(agent.builddirectory)/VSCode-darwin-x64.zip $(agent.builddirectory)/VSCode-darwin.zip
+    displayName: Rename x64 build to it's legacy name
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'))
 
-      VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
-      AZURE_DOCUMENTDB_MASTERKEY="$(builds-docdb-key-readwrite)" \
-      AZURE_STORAGE_ACCESS_KEY="$(ticino-storage-key)" \
-      AZURE_STORAGE_ACCESS_KEY_2="$(vscode-storage-key)" \
-      node build/azure-pipelines/common/createAsset.js \
-        "$ASSET_ID" \
-        archive \
-        "VSCode-$ASSET_ID.zip" \
-        ../VSCode-darwin-$(VSCODE_ARCH).zip
-    displayName: Publish Clients
+  - publish: $(Agent.BuildDirectory)/VSCode-$(ASSET_ID).zip
+    artifact: vscode_client_darwin_$(VSCODE_ARCH)_archive

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -138,19 +138,19 @@ steps:
     condition: and(succeeded(), ne(variables['VSCODE_ARCH'], 'universal'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - download: current
-    artifact: vscode-darwin-x64
+    artifact: unsigned_vscode_client_darwin_x64_archive
     displayName: Download x64 artifact
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
 
   - download: current
-    artifact: vscode-darwin-arm64
+    artifact: unsigned_vscode_client_darwin_arm64_archive
     displayName: Download arm64 artifact
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'universal'))
 
   - script: |
       set -e
-      cp $(Pipeline.Workspace)/vscode-darwin-x64/VSCode-darwin-x64.zip $(agent.builddirectory)/VSCode-darwin-x64.zip
-      cp $(Pipeline.Workspace)/vscode-darwin-arm64/VSCode-darwin-arm64.zip $(agent.builddirectory)/VSCode-darwin-arm64.zip
+      cp $(Pipeline.Workspace)/unsigned_vscode_client_darwin_x64_archive/VSCode-darwin-x64.zip $(agent.builddirectory)/VSCode-darwin-x64.zip
+      cp $(Pipeline.Workspace)/unsigned_vscode_client_darwin_arm64_archive/VSCode-darwin-arm64.zip $(agent.builddirectory)/VSCode-darwin-arm64.zip
       unzip $(agent.builddirectory)/VSCode-darwin-x64.zip -d $(agent.builddirectory)/VSCode-darwin-x64
       unzip $(agent.builddirectory)/VSCode-darwin-arm64.zip -d $(agent.builddirectory)/VSCode-darwin-arm64
       DEBUG=* node build/darwin/create-universal-app.js
@@ -280,26 +280,27 @@ steps:
 
   - script: |
       set -e
-      VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
-      AZURE_DOCUMENTDB_MASTERKEY="$(builds-docdb-key-readwrite)" \
-      AZURE_STORAGE_ACCESS_KEY="$(ticino-storage-key)" \
-      AZURE_STORAGE_ACCESS_KEY_2="$(vscode-storage-key)" \
-      VSCODE_ARCH="$(VSCODE_ARCH)" ./build/azure-pipelines/darwin/publish-server.sh
-    displayName: Publish Servers
+
+      # package Remote Extension Host
+      pushd .. && mv vscode-reh-darwin vscode-server-darwin && zip -Xry vscode-server-darwin.zip vscode-server-darwin && popd
+
+      # package Remote Extension Host (Web)
+      pushd .. && mv vscode-reh-web-darwin vscode-server-darwin-web && zip -Xry vscode-server-darwin-web.zip vscode-server-darwin-web && popd
+    displayName: Prepare to publish servers
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), ne(variables['VSCODE_PUBLISH'], 'false'))
 
   - publish: $(Agent.BuildDirectory)/VSCode-darwin-$(VSCODE_ARCH).zip
-    artifact: vscode-darwin-$(VSCODE_ARCH)
+    artifact: unsigned_vscode_client_darwin_$(VSCODE_ARCH)_archive
     displayName: Publish client archive
     condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
 
   - publish: $(Agent.BuildDirectory)/vscode-server-darwin.zip
-    artifact: vscode-server-darwin-$(VSCODE_ARCH)
+    artifact: vscode_server_darwin_$(VSCODE_ARCH)_archive-unsigned
     displayName: Publish server archive
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), ne(variables['VSCODE_PUBLISH'], 'false'))
 
   - publish: $(Agent.BuildDirectory)/vscode-server-darwin-web.zip
-    artifact: vscode-server-darwin-$(VSCODE_ARCH)-web
+    artifact: vscode_web_darwin_$(VSCODE_ARCH)_archive-unsigned
     displayName: Publish web server archive
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), ne(variables['VSCODE_PUBLISH'], 'false'))