vsx: support buildtime 'extensionPack' handling

The following commit adds improved support for `extensionPack` plugins defined as
builtins. Previously, `extensionPacks` could be referenced as builtins,
but they would later be resolved when the application is first started.
For large packs (such as vscode builtins) this would result in a long
startup time, where extensionPacks would need to be resolved, and then
downloaded. The following commit fixes that problem by allowing
`extensionPack` to be resolved and fetched at buildtime rather than
runtime.

In order to implement the improvement, a new dev-package was created
(`@theia/ovsx-client`) which includes the common logic for the `cli` and
`@theia/vsx-registry` (previously part of the `@theia/vsx-registry`)
extension. This logic which interacts with the registry (fetch, search,
download, determine compatibility) is now used by the `cli` (buildtime),
and `vsx-registry` (runtime). The `@theia/vsx-registry` extension was
refactored and cleaned up as a consequence.

The idea when resolving builtin extensions is the following:
- download extensions like today (master).
- determine if any `extensionPack`(s) exist.
- fetch individual extension ids.
- download latest compatible versions of all individual extension ids.

Signed-off-by: vince-fugnitto <vincent.fugnitto@ericsson.com>
Co-authored-by: marechal-p <paul.marechal@ericsson.com>

configs/root-compilation.tsconfig.json

@@ -144,6 +144,9 @@
     },
     {
       "path": "../packages/property-view/compile.tsconfig.json"
+    },
+    {
+      "path": "../dev-packages/ovsx-client/compile.tsconfig.json"
     }
   ]
 }

dev-packages/cli/compile.tsconfig.json

@@ -14,6 +14,9 @@
     },
     {
       "path": "../application-package/compile.tsconfig.json"
+    },
+    {
+      "path": "../ovsx-client/compile.tsconfig.json"
     }
   ]
 }

dev-packages/cli/package.json

@@ -31,6 +31,7 @@
   "dependencies": {
     "@theia/application-manager": "1.14.0",
     "@theia/application-package": "1.14.0",
+    "@theia/ovsx-client": "1.14.0",
     "@types/chai": "^4.2.7",
     "@types/mkdirp": "^0.5.2",
     "@types/mocha": "^5.2.7",

dev-packages/cli/src/download-plugins.ts

@@ -30,6 +30,7 @@ import * as temp from 'temp';
 import { green, red } from 'colors/safe';
 
 import { promisify } from 'util';
+import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
 const mkdirpAsPromised = promisify<string, mkdirp.Made>(mkdirp);
 const pipelineAsPromised = promisify(stream.pipeline);
 
@@ -50,6 +51,17 @@ export interface DownloadPluginsOptions {
      * Defaults to `false`.
      */
     ignoreErrors?: boolean;
+
+    /**
+     * The supported vscode API version.
+     * Used to determine extension compatibility.
+     */
+    apiVersion?: string;
+
+    /**
+     * The open-vsx registry API url.
+     */
+    apiUrl?: string;
 }
 
 export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise<void> {
@@ -60,6 +72,8 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
     const {
         packed = false,
         ignoreErrors = false,
+        apiVersion = '1.50.0',
+        apiUrl = 'https://open-vsx.org/api'
     } = options;
 
     console.warn('--- downloading plugins ---');
@@ -87,6 +101,20 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
     if (!ignoreErrors && failures.length > 0) {
         throw new Error('Errors downloading some plugins. To make these errors non fatal, re-run with --ignore-errors');
     }
+
+    // Resolve extension pack plugins.
+    const ids = await getAllExtensionPackIds(pluginsDir);
+    if (ids.length) {
+        const client = new OVSXClient({ apiVersion, apiUrl });
+        ids.forEach(async id => {
+            const extension = await client.getLatestCompatibleExtensionVersion(id);
+            const downloadUrl = extension?.files.download;
+            if (downloadUrl) {
+                await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed);
+            }
+        });
+    }
+
 }
 
 /**
@@ -189,3 +217,60 @@ export function xfetch(url: string, options?: RequestInit): Promise<Response> {
     }
     return fetch(url, proxiedOptions);
 }
+
+/**
+ * Get the list of all available ids referenced by extension packs.
+ * @param pluginDir the plugin directory.
+ * @returns the list of all referenced extension pack ids.
+ */
+async function getAllExtensionPackIds(pluginDir: string): Promise<string[]> {
+    const extensions = await getPackageFiles(pluginDir);
+    const extensionIds: string[] = [];
+    const ids = await Promise.all(extensions.map(ext => getExtensionPackIds(ext)));
+    ids.forEach(id => {
+        extensionIds.push(...id);
+    });
+    return extensionIds;
+}
+
+/**
+ * Walk the plugin directory collecting available extension paths.
+ * @param dirPath the plugin directory
+ * @returns the list of extension paths.
+ */
+async function getPackageFiles(dirPath: string): Promise<string[]> {
+    let fileList: string[] = [];
+    const files = await fs.readdir(dirPath);
+
+    // Recursively fetch the list of extension `package.json` files.
+    for (const file of files) {
+        const filePath = path.join(dirPath, file);
+        if ((await fs.stat(filePath)).isDirectory()) {
+            fileList = [...fileList, ...(await getPackageFiles(filePath))];
+        } else if ((path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules'))) {
+            fileList.push(filePath);
+        }
+    }
+
+    return fileList;
+}
+
+/**
+ * Get the list of extension ids referenced by the extension pack.
+ * @param extPath the individual extension path.
+ * @returns the list of individual extension ids.
+ */
+async function getExtensionPackIds(extPath: string): Promise<string[]> {
+    const ids = new Set<string>();
+    const content = await fs.readFile(extPath, 'utf-8');
+    const json = JSON.parse(content);
+
+    // The `extensionPack` object.
+    const extensionPack = json.extensionPack as string[];
+    for (const ext in extensionPack) {
+        if (ext !== undefined) {
+            ids.add(extensionPack[ext]);
+        }
+    }
+    return Array.from(ids);
+}

dev-packages/cli/src/theia.ts

@@ -162,6 +162,16 @@ function rebuildCommand(command: string, target: ApplicationProps.Target): yargs
                     describe: 'Ignore errors while downloading plugins',
                     boolean: true,
                     default: false,
+                },
+                'api-version': {
+                    alias: 'v',
+                    describe: 'Supported API version for plugins',
+                    default: '1.50.0'
+                },
+                'api-url': {
+                    alias: 'u',
+                    describe: 'Open-VSX Registry API URL',
+                    default: 'https://open-vsx.org/api'
                 }
             },
             handler: async args => {

dev-packages/ovsx-client/.eslintrc.js

@@ -0,0 +1,10 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+    extends: [
+        '../../configs/build.eslintrc.json'
+    ],
+    parserOptions: {
+        tsconfigRootDir: __dirname,
+        project: 'compile.tsconfig.json'
+    }
+};

dev-packages/ovsx-client/README.md

@@ -0,0 +1,31 @@
+<div align='center'>
+
+<br />
+
+<img src='https://raw.githubusercontent.com/eclipse-theia/theia/master/logo/theia.svg?sanitize=true' alt='theia-ext-logo' width='100px' />
+
+<h2>ECLIPSE THEIA - OVSX CLIENT</h2>
+
+<hr />
+
+</div>
+
+## Description
+
+The `@theia/ovsx-client` package is used to interact with `open-vsx` through its REST APIs.
+The package allows clients to fetch extensions and their metadata, search the registry, and
+includes the necessary logic to determine compatibility based on a provided supported API version.
+
+## Additional Information
+
+- [Theia - GitHub](https://github.com/eclipse-theia/theia)
+- [Theia - Website](https://theia-ide.org/)
+
+## License
+
+- [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/)
+- [一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp)
+
+## Trademark
+"Theia" is a trademark of the Eclipse Foundation
+https://www.eclipse.org/theia

dev-packages/ovsx-client/compile.tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../configs/base.tsconfig",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "lib"
+  },
+  "include": [
+    "src"
+  ],
+  "references": []
+}

dev-packages/ovsx-client/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@theia/ovsx-client",
+  "version": "1.14.0",
+  "description": "Theia Open-VSX Client",
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/eclipse-theia/theia.git"
+  },
+  "bugs": {
+    "url": "https://github.com/eclipse-theia/theia/issues"
+  },
+  "homepage": "https://github.com/eclipse-theia/theia",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "scripts": {
+    "lint": "theiaext lint",
+    "build": "theiaext build",
+    "watch": "theiaext watch",
+    "clean": "theiaext clean",
+    "test": "theiaext test"
+  },
+  "dependencies": {
+    "@types/bent": "^7.0.1",
+    "bent": "^7.1.0",
+    "semver": "^5.4.1"
+  }
+}

packages/vsx-registry/src/common/vsx-api-version-provider.ts → dev-packages/ovsx-client/src/index.ts

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2020 Ericsson and others.
+ * Copyright (C) 2021 Ericsson and others.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v. 2.0 which is available at
@@ -14,8 +14,5 @@
  * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
  ********************************************************************************/
 
-export const VSXApiVersionProvider = Symbol('VSXApiVersionProvider');
-
-export interface VSXApiVersionProvider {
-    getApiVersion(): string;
-}
+export * from './ovsx-client';
+export * from './ovsx-types';

dev-packages/ovsx-client/src/ovsx-client.spec.ts

@@ -0,0 +1,112 @@
+/********************************************************************************
+ * Copyright (C) 2020 Ericsson and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import * as chai from 'chai';
+import { OVSXClient } from './ovsx-client';
+import { VSXSearchParam } from './ovsx-types';
+
+const expect = chai.expect;
+
+describe('OVSX Client', () => {
+
+    const apiUrl = 'https://open-vsx.org/api';
+    const apiVersion = '1.40.0';
+
+    let client: OVSXClient;
+
+    before(() => {
+        client = new OVSXClient({
+            apiVersion,
+            apiUrl
+        });
+    });
+
+    describe('isEngineValid', () => {
+
+        it('should return \'true\' for a compatible engine', () => {
+            const a: boolean = client['isEngineSupported']('^1.20.0');
+            const b: boolean = client['isEngineSupported']('^1.40.0');
+            expect(a).to.eq(true);
+            expect(b).to.eq(true);
+        });
+
+        it('should return \'true\' for the wildcard \'*\' engine', () => {
+            const valid: boolean = client['isEngineSupported']('*');
+            expect(valid).to.eq(true);
+        });
+
+        it('should return \'false\' for a incompatible engine', () => {
+            const valid: boolean = client['isEngineSupported']('^1.50.0');
+            expect(valid).to.eq(false);
+        });
+
+        it('should return \'false\' for an undefined engine', () => {
+            const valid: boolean = client['isEngineSupported']();
+            expect(valid).to.eq(false);
+        });
+
+    });
+
+    describe('#buildSearchUri', () => {
+
+        it('should correctly build the search URI with the single `query` parameter present', async () => {
+            const expected = 'https://open-vsx.org/api/-/search?query=javascript';
+            const param: VSXSearchParam = {
+                query: 'javascript',
+            };
+            const query = await client['buildSearchUri'](param);
+            expect(query).to.eq(expected);
+        });
+
+        it('should correctly build the search URI with the multiple search parameters present', async () => {
+            let expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&includeAllVersions=true';
+            let param: VSXSearchParam = {
+                query: 'javascript',
+                category: 'languages',
+                size: 20,
+                offset: 10,
+                includeAllVersions: true,
+            };
+            let query = await client['buildSearchUri'](param);
+            expect(query).to.eq(expected);
+
+            expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&sortOrder=desc&sortBy=relevance&includeAllVersions=true';
+            param = {
+                query: 'javascript',
+                category: 'languages',
+                size: 20,
+                offset: 10,
+                sortOrder: 'desc',
+                sortBy: 'relevance',
+                includeAllVersions: true
+            };
+            query = await client['buildSearchUri'](param);
+            expect(query).to.eq(expected);
+        });
+
+    });
+
+    describe('#isVersionLTE', () => {
+
+        it('should determine if v1 is less than or equal to v2', () => {
+            expect(client['isVersionLTE']('1.40.0', '1.50.0')).equal(true, 'should be satisfied since v1 is less than v2');
+            expect(client['isVersionLTE']('1.50.0', '1.50.0')).equal(true, 'should be satisfied since v1 and v2 are equal');
+            expect(client['isVersionLTE']('2.0.2', '2.0.1')).equal(false, 'should not be satisfied since v1 is greater than v2');
+        });
+
+    });
+
+});