Validate vsix urls in gitpod.yaml

Author: jeanp413

extensions/gitpod-web/package.json

@@ -502,12 +502,18 @@
   },
   "devDependencies": {
     "@types/node": "^10.12.21",
-    "@types/node-fetch": "^2.5.8"
+    "@types/node-fetch": "^2.5.12",
+    "@types/uuid": "8.0.0",
+    "@types/yauzl": "^2.9.1",
+    "@types/yazl": "^2.4.2"
   },
   "dependencies": {
     "gitpod-shared": "0.0.1",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "^2.6.5",
+    "uuid": "8.1.0",
     "vscode-jsonrpc": "^5.0.1",
-    "vscode-nls": "^5.0.0"
+    "vscode-nls": "^5.0.0",
+    "yauzl": "^2.9.2",
+    "yazl": "^2.4.3"
   }
 }

extensions/gitpod-web/src/extension.ts

@@ -8,6 +8,8 @@
 import * as workspaceInstance from '@gitpod/gitpod-protocol/lib/workspace-instance';
 import * as grpc from '@grpc/grpc-js';
 import * as fs from 'fs';
+import * as os from 'os';
+import * as uuid from 'uuid';
 import { GitpodPluginModel, GitpodExtensionContext, setupGitpodContext, registerTasks, registerIpcHookCli } from 'gitpod-shared';
 import { GetTokenRequest } from '@gitpod/supervisor-api-grpc/lib/token_pb';
 import { PortsStatus, ExposedPortInfo, PortsStatusRequest, PortsStatusResponse, PortAutoExposure, PortVisibility, OnPortExposedAction } from '@gitpod/supervisor-api-grpc/lib/status_pb';
@@ -19,7 +21,9 @@ import * as path from 'path';
 import { URL } from 'url';
 import * as util from 'util';
 import * as vscode from 'vscode';
-import { ThrottledDelayer } from './async';
+import { ThrottledDelayer } from './util/async';
+import { download } from './util/download';
+import { getManifest } from './util/extensionManagmentUtill';
 
 let gitpodContext: GitpodExtensionContext | undefined;
 export async function activate(context: vscode.ExtensionContext) {
@@ -185,7 +189,7 @@ export function registerAuth(context: GitpodExtensionContext): void {
 			if (!userResponse.ok) {
 				throw new Error(`Getting GitHub account info failed: ${userResponse.statusText}`);
 			}
-			const user: { id: string, login: string } = await userResponse.json();
+			const user = await (userResponse.json() as Promise<{ id: string, login: string }>);
 			return {
 				id: user.id,
 				accountName: user.login
@@ -774,7 +778,7 @@ interface IOpenVSXQueryResult {
 	extensions: IOpenVSXExtensionsMetadata[];
 }
 
-async function validateExtensions(extensionsToValidate: { id: string, version?: string }[], token: vscode.CancellationToken) {
+async function validateExtensions(extensionsToValidate: { id: string, version?: string }[], linkToValidate: string[], token: vscode.CancellationToken) {
 	const allUserExtensions = vscode.extensions.all.filter(ext => !ext.packageJSON['isBuiltin'] && !ext.packageJSON['isUserBuiltin']);
 
 	const lookup = new Set<string>(extensionsToValidate.map(({ id }) => id));
@@ -795,7 +799,8 @@ async function validateExtensions(extensionsToValidate: { id: string, version?:
 			return {
 				extensions: [],
 				missingMachined: [],
-				uninstalled: []
+				uninstalled: [],
+				links: []
 			};
 		}
 	}
@@ -806,21 +811,21 @@ async function validateExtensions(extensionsToValidate: { id: string, version?:
 			`${process.env.VSX_REGISTRY_URL || 'https://open-vsx.org'}/api/-/query`,
 			{
 				method: 'POST',
-				timeout: 5000,
 				headers: {
 					'Content-Type': 'application/json',
 					'Accept': 'application/json'
 				},
 				body: JSON.stringify({
 					extensionId: id
-				})
+				}),
+				timeout: 2000
 			}
 		).then(resp => {
 			if (!resp.ok) {
 				console.error('Failed to query open-vsx while validating gitpod.yml');
 				return undefined;
 			}
-			return resp.json();
+			return resp.json() as Promise<IOpenVSXQueryResult>;
 		}, e => {
 			console.error('Fetch failed while querying open-vsx', e);
 			return undefined;
@@ -837,17 +842,40 @@ async function validateExtensions(extensionsToValidate: { id: string, version?:
 			return {
 				extensions: [],
 				missingMachined: [],
-				uninstalled: []
+				uninstalled: [],
+				links: []
 			};
 		}
 	}
 
-	// TODO: validate links
+	const links = new Set<string>();
+	for (const link of linkToValidate) {
+		const downloadPath = path.join(os.tmpdir(), uuid.v4());
+		try {
+			await download(link, downloadPath, token, 10000);
+			const manifest = await getManifest(downloadPath);
+			if (manifest.engines?.vscode) {
+				links.add(link);
+			}
+		} catch (error) {
+			console.error('Failed to validate vsix url', error);
+		}
+
+		if (token.isCancellationRequested) {
+			return {
+				extensions: [],
+				missingMachined: [],
+				uninstalled: [],
+				links: []
+			};
+		}
+	}
 
 	return {
 		extensions: [...validatedExtensions],
 		missingMachined: [...missingMachined],
-		uninstalled: [...uninstalled]
+		uninstalled: [...uninstalled],
+		links: [...links]
 	};
 }
 
@@ -926,10 +954,7 @@ export function registerExtensionManagement(context: GitpodExtensionContext): vo
 			}
 			try {
 				const toLink = new Map<string, vscode.Range>();
-				const toFind = new Map<string, {
-					version?: string,
-					range: vscode.Range
-				}>();
+				const toFind = new Map<string, { version?: string, range: vscode.Range }>();
 				let document: vscode.TextDocument | undefined;
 				try {
 					document = await vscode.workspace.openTextDocument(gitpodFileUri);
@@ -994,7 +1019,8 @@ export function registerExtensionManagement(context: GitpodExtensionContext): vo
 					}
 
 					const extensionsToValidate = [...toFind.entries()].map(([id, { version }]) => ({ id, version }));
-					const result = await validateExtensions(extensionsToValidate, token);
+					const linksToValidate = [...toLink.keys()];
+					const result = await validateExtensions(extensionsToValidate, linksToValidate, token);
 
 					if (token.isCancellationRequested) {
 						return;
@@ -1016,14 +1042,14 @@ export function registerExtensionManagement(context: GitpodExtensionContext): vo
 						pushDiagnostic(diagnostic);
 					}
 
-					// for (const link of result.links) {
-					// 	toLink.delete(link);
-					// }
-					// for (const [link, range] of toLink) {
-					// 	const diagnostic = new vscode.Diagnostic(range, link + invalidVSIXLinkMessageSuffix, vscode.DiagnosticSeverity.Error);
-					// 	diagnostic.source = 'gitpod';
-					// 	pushDiagnostic(diagnostic);
-					// }
+					for (const link of result.links) {
+						toLink.delete(link);
+					}
+					for (const [link, range] of toLink) {
+						const diagnostic = new vscode.Diagnostic(range, link + invalidVSIXLinkMessageSuffix, vscode.DiagnosticSeverity.Error);
+						diagnostic.source = 'gitpod';
+						pushDiagnostic(diagnostic);
+					}
 
 					for (const id of result.missingMachined) {
 						const diagnostic = new vscode.Diagnostic(new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)), id + missingExtensionMessageSuffix, vscode.DiagnosticSeverity.Warning);
@@ -1158,13 +1184,9 @@ export function registerExtensionManagement(context: GitpodExtensionContext): vo
 			return codeActions;
 		}
 	}));
+
 	validateGitpodFile();
 	context.subscriptions.push(gitpodDiagnostics);
-	context.subscriptions.push(vscode.workspace.onDidChangeTextDocument(e => {
-		if (e.document.uri.toString() === gitpodFileUri.toString()) {
-			validateGitpodFile();
-		}
-	}));
 	const gitpodFileWatcher = vscode.workspace.createFileSystemWatcher(gitpodFileUri.fsPath);
 	context.subscriptions.push(gitpodFileWatcher);
 	context.subscriptions.push(gitpodFileWatcher.onDidCreate(() => validateGitpodFile()));

extensions/gitpod-web/src/async.ts renamed to extensions/gitpod-web/src/util/async.ts

@@ -3,6 +3,49 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { CancellationToken, CancellationTokenSource } from 'vscode';
+
+export interface CancelablePromise<T> extends Promise<T> {
+	cancel(): void;
+}
+
+export function createCancelablePromise<T>(callback: (token: CancellationToken) => Promise<T>): CancelablePromise<T> {
+	const source = new CancellationTokenSource();
+
+	const thenable = callback(source.token);
+	const promise = new Promise<T>((resolve, reject) => {
+		const subscription = source.token.onCancellationRequested(() => {
+			subscription.dispose();
+			source.dispose();
+			reject(new Error('Canceled Promise'));
+		});
+		Promise.resolve(thenable).then(value => {
+			subscription.dispose();
+			source.dispose();
+			resolve(value);
+		}, err => {
+			subscription.dispose();
+			source.dispose();
+			reject(err);
+		});
+	});
+
+	return <CancelablePromise<T>>new class {
+		cancel() {
+			source.cancel();
+		}
+		then<TResult1 = T, TResult2 = never>(resolve?: ((value: T) => TResult1 | Promise<TResult1>) | undefined | null, reject?: ((reason: any) => TResult2 | Promise<TResult2>) | undefined | null): Promise<TResult1 | TResult2> {
+			return promise.then(resolve, reject);
+		}
+		catch<TResult = never>(reject?: ((reason: any) => TResult | Promise<TResult>) | undefined | null): Promise<T | TResult> {
+			return this.then(undefined, reject);
+		}
+		finally(onfinally?: (() => void) | undefined | null): Promise<T> {
+			return promise.finally(onfinally);
+		}
+	};
+}
+
 export interface ITask<T> {
 	(): T;
 }
@@ -77,6 +120,15 @@ export class Throttler<T> {
 	}
 }
 
+export class Sequencer {
+
+	private current: Promise<unknown> = Promise.resolve(null);
+
+	queue<T>(promiseTask: ITask<Promise<T>>): Promise<T> {
+		return this.current = this.current.then(() => promiseTask(), () => promiseTask());
+	}
+}
+
 /**
  * A helper to delay execution of a task that is being requested often.
  *

extensions/gitpod-web/src/util/download.ts

@@ -0,0 +1,59 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Gitpod. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as fs from 'fs';
+import * as https from 'https';
+import * as http from 'http';
+import { basename } from 'path';
+import { URL } from 'url';
+import { createGunzip } from 'zlib';
+import * as stream from 'stream';
+import * as vscode from 'vscode';
+
+export function download(url: string, dest: string, token: vscode.CancellationToken, timeout?: number): Promise<void> {
+	const uri = new URL(url);
+	if (!dest) {
+		dest = basename(uri.pathname);
+	}
+	const pkg = url.toLowerCase().startsWith('https:') ? https : http;
+
+	return new Promise((resolve, reject) => {
+		const request = pkg.get(uri.href).on('response', (res: http.IncomingMessage) => {
+			if (res.statusCode === 200) {
+				const file = fs.createWriteStream(dest, { flags: 'wx' });
+				res.on('end', () => {
+					file.end();
+					// console.log(`${uri.pathname} downloaded to: ${path}`)
+					resolve();
+				}).on('error', (err: any) => {
+					file.destroy();
+					fs.unlink(dest, () => reject(err));
+				});
+
+				let dataStream: stream.Readable = res;
+				if (res.headers['content-encoding'] === 'gzip') {
+					dataStream = res.pipe(createGunzip());
+				}
+				dataStream.pipe(file);
+			} else if (res.statusCode === 302 || res.statusCode === 301) {
+				// Recursively follow redirects, only a 200 will resolve.
+				download(res.headers.location!, dest, token, timeout).then(() => resolve());
+			} else {
+				reject(new Error(`Download request failed, response status: ${res.statusCode} ${res.statusMessage}`));
+			}
+		});
+
+		if (timeout) {
+			request.setTimeout(timeout, () => {
+				request.abort();
+				reject(new Error(`Request timeout after ${timeout / 1000.0}s`));
+			});
+		}
+
+		token.onCancellationRequested(() => {
+			request.abort();
+			reject(new Error(`Request cancelled`));
+		});
+	});
+}

extensions/gitpod-web/src/util/extensionManagmentUtill.ts

@@ -0,0 +1,44 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { buffer } from './zip';
+
+export type ExtensionKind = 'ui' | 'workspace' | 'web';
+
+export interface IExtensionManifest {
+	readonly name: string;
+	readonly displayName?: string;
+	readonly publisher: string;
+	readonly version: string;
+	readonly engines: { readonly vscode: string };
+	readonly description?: string;
+	readonly main?: string;
+	readonly browser?: string;
+	readonly icon?: string;
+	readonly categories?: string[];
+	readonly keywords?: string[];
+	readonly activationEvents?: string[];
+	readonly extensionDependencies?: string[];
+	readonly extensionPack?: string[];
+	readonly extensionKind?: ExtensionKind | ExtensionKind[];
+	readonly contributes?: any;
+	readonly repository?: { url: string; };
+	readonly bugs?: { url: string; };
+	readonly enableProposedApi?: boolean;
+	readonly api?: string;
+	readonly scripts?: { [key: string]: string; };
+	readonly capabilities?: any;
+}
+
+export function getManifest(vsix: string): Promise<IExtensionManifest> {
+	return buffer(vsix, 'extension/package.json')
+		.then(buffer => {
+			try {
+				return JSON.parse(buffer.toString('utf8'));
+			} catch (err) {
+				throw new Error('VSIX invalid: package.json is not a JSON file.');
+			}
+		});
+}