From 8c19823d4be1f2878493f7b0a6bf3f40c4bcbc45 Mon Sep 17 00:00:00 2001 From: vince-fugnitto Date: Thu, 29 Apr 2021 12:41:52 -0400 Subject: [PATCH] vsx: support buildtime 'extensionPack' handling The following commit adds improved support for `extensionPack` plugins defined as builtins. Previously, `extensionPacks` could be referenced as builtins, but they would later be resolved when the application is first started. For large packs (such as vscode builtins) this would result in a long startup time, where extensionPacks would need to be resolved, and then downloaded. The following commit fixes that problem by allowing `extensionPack` to be resolved and fetched at buildtime rather than runtime. In order to implement the improvement, a new dev-package was created (`@theia/ovsx-client`) which includes the common logic for the `cli` and `@theia/vsx-registry` (previously part of the `@theia/vsx-registry`) extension. This logic which interacts with the registry (fetch, search, download, determine compatibility) is now used by the `cli` (buildtime), and `vsx-registry` (runtime). The `@theia/vsx-registry` extension was refactored and cleaned up as a consequence. The idea when resolving builtin extensions is the following: - download extensions like today (master). - determine if any `extensionPack`(s) exist. - fetch individual extension ids. - download latest compatible versions of all individual extension ids. Signed-off-by: vince-fugnitto Co-authored-by: marechal-p --- configs/root-compilation.tsconfig.json | 3 + dev-packages/cli/compile.tsconfig.json | 3 + dev-packages/cli/package.json | 1 + dev-packages/cli/src/download-plugins.ts | 85 +++++++++ dev-packages/cli/src/theia.ts | 10 ++ dev-packages/ovsx-client/.eslintrc.js | 10 ++ dev-packages/ovsx-client/README.md | 31 ++++ .../ovsx-client/compile.tsconfig.json | 12 ++ dev-packages/ovsx-client/package.json | 35 ++++ .../ovsx-client/src/index.ts | 9 +- .../ovsx-client/src/ovsx-client.spec.ts | 112 ++++++++++++ .../ovsx-client/src/ovsx-client.ts | 80 +++------ .../ovsx-client/src/ovsx-types.ts | 26 +++ .../src/common/plugin-vscode-types.ts | 1 + packages/vsx-registry/compile.tsconfig.json | 3 + packages/vsx-registry/package.json | 2 +- .../src/browser/ovsx-async-client.ts | 39 ++++ .../vsx-api-version-provider-frontend-impl.ts | 38 ---- .../src/browser/vsx-extension.tsx | 2 +- .../src/browser/vsx-extensions-model.ts | 17 +- .../browser/vsx-registry-frontend-module.ts | 21 ++- .../src/common/vsx-registry-api.spec.ts | 168 ------------------ .../vsx-api-version-provider-backend-impl.ts | 28 --- .../src/node/vsx-extension-resolver.ts | 8 +- .../src/node/vsx-registry-backend-module.ts | 20 +-- tsconfig.json | 3 + 26 files changed, 442 insertions(+), 325 deletions(-) create mode 100644 dev-packages/ovsx-client/.eslintrc.js create mode 100644 dev-packages/ovsx-client/README.md create mode 100644 dev-packages/ovsx-client/compile.tsconfig.json create mode 100644 dev-packages/ovsx-client/package.json rename packages/vsx-registry/src/common/vsx-api-version-provider.ts => dev-packages/ovsx-client/src/index.ts (80%) create mode 100644 dev-packages/ovsx-client/src/ovsx-client.spec.ts rename packages/vsx-registry/src/common/vsx-registry-api.ts => dev-packages/ovsx-client/src/ovsx-client.ts (75%) rename packages/vsx-registry/src/common/vsx-registry-types.ts => dev-packages/ovsx-client/src/ovsx-types.ts (87%) create mode 100644 packages/vsx-registry/src/browser/ovsx-async-client.ts delete mode 100644 packages/vsx-registry/src/browser/vsx-api-version-provider-frontend-impl.ts delete mode 100644 packages/vsx-registry/src/common/vsx-registry-api.spec.ts delete mode 100644 packages/vsx-registry/src/node/vsx-api-version-provider-backend-impl.ts diff --git a/configs/root-compilation.tsconfig.json b/configs/root-compilation.tsconfig.json index 6ca6c16407c40..e4c89e349dcd1 100644 --- a/configs/root-compilation.tsconfig.json +++ b/configs/root-compilation.tsconfig.json @@ -144,6 +144,9 @@ }, { "path": "../packages/property-view/compile.tsconfig.json" + }, + { + "path": "../dev-packages/ovsx-client/compile.tsconfig.json" } ] } diff --git a/dev-packages/cli/compile.tsconfig.json b/dev-packages/cli/compile.tsconfig.json index 9686bb1b2ef0c..4061eff43ff03 100644 --- a/dev-packages/cli/compile.tsconfig.json +++ b/dev-packages/cli/compile.tsconfig.json @@ -14,6 +14,9 @@ }, { "path": "../application-package/compile.tsconfig.json" + }, + { + "path": "../ovsx-client/compile.tsconfig.json" } ] } diff --git a/dev-packages/cli/package.json b/dev-packages/cli/package.json index ca97202ac1181..84d301ca31e01 100644 --- a/dev-packages/cli/package.json +++ b/dev-packages/cli/package.json @@ -31,6 +31,7 @@ "dependencies": { "@theia/application-manager": "1.14.0", "@theia/application-package": "1.14.0", + "@theia/ovsx-client": "1.14.0", "@types/chai": "^4.2.7", "@types/mkdirp": "^0.5.2", "@types/mocha": "^5.2.7", diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index 809a52321a9ae..8f873d5c7208e 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -30,6 +30,7 @@ import * as temp from 'temp'; import { green, red } from 'colors/safe'; import { promisify } from 'util'; +import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client'; const mkdirpAsPromised = promisify(mkdirp); const pipelineAsPromised = promisify(stream.pipeline); @@ -50,6 +51,17 @@ export interface DownloadPluginsOptions { * Defaults to `false`. */ ignoreErrors?: boolean; + + /** + * The supported vscode API version. + * Used to determine extension compatibility. + */ + apiVersion?: string; + + /** + * The open-vsx registry API url. + */ + apiUrl?: string; } export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise { @@ -60,6 +72,8 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = const { packed = false, ignoreErrors = false, + apiVersion = '1.50.0', + apiUrl = 'https://open-vsx.org/api' } = options; console.warn('--- downloading plugins ---'); @@ -87,6 +101,20 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = if (!ignoreErrors && failures.length > 0) { throw new Error('Errors downloading some plugins. To make these errors non fatal, re-run with --ignore-errors'); } + + // Resolve extension pack plugins. + const ids = await getAllExtensionPackIds(pluginsDir); + if (ids.length) { + const client = new OVSXClient({ apiVersion, apiUrl }); + ids.forEach(async id => { + const extension = await client.getLatestCompatibleExtensionVersion(id); + const downloadUrl = extension?.files.download; + if (downloadUrl) { + await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed); + } + }); + } + } /** @@ -189,3 +217,60 @@ export function xfetch(url: string, options?: RequestInit): Promise { } return fetch(url, proxiedOptions); } + +/** + * Get the list of all available ids referenced by extension packs. + * @param pluginDir the plugin directory. + * @returns the list of all referenced extension pack ids. + */ +async function getAllExtensionPackIds(pluginDir: string): Promise { + const extensions = await getPackageFiles(pluginDir); + const extensionIds: string[] = []; + const ids = await Promise.all(extensions.map(ext => getExtensionPackIds(ext))); + ids.forEach(id => { + extensionIds.push(...id); + }); + return extensionIds; +} + +/** + * Walk the plugin directory collecting available extension paths. + * @param dirPath the plugin directory + * @returns the list of extension paths. + */ +async function getPackageFiles(dirPath: string): Promise { + let fileList: string[] = []; + const files = await fs.readdir(dirPath); + + // Recursively fetch the list of extension `package.json` files. + for (const file of files) { + const filePath = path.join(dirPath, file); + if ((await fs.stat(filePath)).isDirectory()) { + fileList = [...fileList, ...(await getPackageFiles(filePath))]; + } else if ((path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules'))) { + fileList.push(filePath); + } + } + + return fileList; +} + +/** + * Get the list of extension ids referenced by the extension pack. + * @param extPath the individual extension path. + * @returns the list of individual extension ids. + */ +async function getExtensionPackIds(extPath: string): Promise { + const ids = new Set(); + const content = await fs.readFile(extPath, 'utf-8'); + const json = JSON.parse(content); + + // The `extensionPack` object. + const extensionPack = json.extensionPack as string[]; + for (const ext in extensionPack) { + if (ext !== undefined) { + ids.add(extensionPack[ext]); + } + } + return Array.from(ids); +} diff --git a/dev-packages/cli/src/theia.ts b/dev-packages/cli/src/theia.ts index 161518aa45059..7fb75e953ddf8 100644 --- a/dev-packages/cli/src/theia.ts +++ b/dev-packages/cli/src/theia.ts @@ -162,6 +162,16 @@ function rebuildCommand(command: string, target: ApplicationProps.Target): yargs describe: 'Ignore errors while downloading plugins', boolean: true, default: false, + }, + 'api-version': { + alias: 'v', + describe: 'Supported API version for plugins', + default: '1.50.0' + }, + 'api-url': { + alias: 'u', + describe: 'Open-VSX Registry API URL', + default: 'https://open-vsx.org/api' } }, handler: async args => { diff --git a/dev-packages/ovsx-client/.eslintrc.js b/dev-packages/ovsx-client/.eslintrc.js new file mode 100644 index 0000000000000..be9cf1a1b3dff --- /dev/null +++ b/dev-packages/ovsx-client/.eslintrc.js @@ -0,0 +1,10 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + extends: [ + '../../configs/build.eslintrc.json' + ], + parserOptions: { + tsconfigRootDir: __dirname, + project: 'compile.tsconfig.json' + } +}; diff --git a/dev-packages/ovsx-client/README.md b/dev-packages/ovsx-client/README.md new file mode 100644 index 0000000000000..4e01943b0270e --- /dev/null +++ b/dev-packages/ovsx-client/README.md @@ -0,0 +1,31 @@ +
+ +
+ +theia-ext-logo + +

ECLIPSE THEIA - OVSX CLIENT

+ +
+ +
+ +## Description + +The `@theia/ovsx-client` package is used to interact with `open-vsx` through its REST APIs. +The package allows clients to fetch extensions and their metadata, search the registry, and +includes the necessary logic to determine compatibility based on a provided supported API version. + +## Additional Information + +- [Theia - GitHub](https://github.com/eclipse-theia/theia) +- [Theia - Website](https://theia-ide.org/) + +## License + +- [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/) +- [δΈ€ (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp) + +## Trademark +"Theia" is a trademark of the Eclipse Foundation +https://www.eclipse.org/theia diff --git a/dev-packages/ovsx-client/compile.tsconfig.json b/dev-packages/ovsx-client/compile.tsconfig.json new file mode 100644 index 0000000000000..b973ddbc673a2 --- /dev/null +++ b/dev-packages/ovsx-client/compile.tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../configs/base.tsconfig", + "compilerOptions": { + "composite": true, + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ], + "references": [] +} diff --git a/dev-packages/ovsx-client/package.json b/dev-packages/ovsx-client/package.json new file mode 100644 index 0000000000000..4e5e89205f2c7 --- /dev/null +++ b/dev-packages/ovsx-client/package.json @@ -0,0 +1,35 @@ +{ + "name": "@theia/ovsx-client", + "version": "1.14.0", + "description": "Theia Open-VSX Client", + "publishConfig": { + "access": "public" + }, + "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0", + "repository": { + "type": "git", + "url": "https://github.com/eclipse-theia/theia.git" + }, + "bugs": { + "url": "https://github.com/eclipse-theia/theia/issues" + }, + "homepage": "https://github.com/eclipse-theia/theia", + "files": [ + "lib", + "src" + ], + "main": "lib/index.js", + "typings": "lib/index.d.ts", + "scripts": { + "lint": "theiaext lint", + "build": "theiaext build", + "watch": "theiaext watch", + "clean": "theiaext clean", + "test": "theiaext test" + }, + "dependencies": { + "@types/bent": "^7.0.1", + "bent": "^7.1.0", + "semver": "^5.4.1" + } +} diff --git a/packages/vsx-registry/src/common/vsx-api-version-provider.ts b/dev-packages/ovsx-client/src/index.ts similarity index 80% rename from packages/vsx-registry/src/common/vsx-api-version-provider.ts rename to dev-packages/ovsx-client/src/index.ts index 2f4a8b9e31fec..6b18bfe887f91 100644 --- a/packages/vsx-registry/src/common/vsx-api-version-provider.ts +++ b/dev-packages/ovsx-client/src/index.ts @@ -1,5 +1,5 @@ /******************************************************************************** - * Copyright (C) 2020 Ericsson and others. + * Copyright (C) 2021 Ericsson and others. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0 which is available at @@ -14,8 +14,5 @@ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 ********************************************************************************/ -export const VSXApiVersionProvider = Symbol('VSXApiVersionProvider'); - -export interface VSXApiVersionProvider { - getApiVersion(): string; -} +export * from './ovsx-client'; +export * from './ovsx-types'; diff --git a/dev-packages/ovsx-client/src/ovsx-client.spec.ts b/dev-packages/ovsx-client/src/ovsx-client.spec.ts new file mode 100644 index 0000000000000..4d7300c97b3ae --- /dev/null +++ b/dev-packages/ovsx-client/src/ovsx-client.spec.ts @@ -0,0 +1,112 @@ +/******************************************************************************** + * Copyright (C) 2020 Ericsson and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import * as chai from 'chai'; +import { OVSXClient } from './ovsx-client'; +import { VSXSearchParam } from './ovsx-types'; + +const expect = chai.expect; + +describe('OVSX Client', () => { + + const apiUrl = 'https://open-vsx.org/api'; + const apiVersion = '1.40.0'; + + let client: OVSXClient; + + before(() => { + client = new OVSXClient({ + apiVersion, + apiUrl + }); + }); + + describe('isEngineValid', () => { + + it('should return \'true\' for a compatible engine', () => { + const a: boolean = client['isEngineSupported']('^1.20.0'); + const b: boolean = client['isEngineSupported']('^1.40.0'); + expect(a).to.eq(true); + expect(b).to.eq(true); + }); + + it('should return \'true\' for the wildcard \'*\' engine', () => { + const valid: boolean = client['isEngineSupported']('*'); + expect(valid).to.eq(true); + }); + + it('should return \'false\' for a incompatible engine', () => { + const valid: boolean = client['isEngineSupported']('^1.50.0'); + expect(valid).to.eq(false); + }); + + it('should return \'false\' for an undefined engine', () => { + const valid: boolean = client['isEngineSupported'](); + expect(valid).to.eq(false); + }); + + }); + + describe('#buildSearchUri', () => { + + it('should correctly build the search URI with the single `query` parameter present', async () => { + const expected = 'https://open-vsx.org/api/-/search?query=javascript'; + const param: VSXSearchParam = { + query: 'javascript', + }; + const query = await client['buildSearchUri'](param); + expect(query).to.eq(expected); + }); + + it('should correctly build the search URI with the multiple search parameters present', async () => { + let expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&includeAllVersions=true'; + let param: VSXSearchParam = { + query: 'javascript', + category: 'languages', + size: 20, + offset: 10, + includeAllVersions: true, + }; + let query = await client['buildSearchUri'](param); + expect(query).to.eq(expected); + + expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&sortOrder=desc&sortBy=relevance&includeAllVersions=true'; + param = { + query: 'javascript', + category: 'languages', + size: 20, + offset: 10, + sortOrder: 'desc', + sortBy: 'relevance', + includeAllVersions: true + }; + query = await client['buildSearchUri'](param); + expect(query).to.eq(expected); + }); + + }); + + describe('#isVersionLTE', () => { + + it('should determine if v1 is less than or equal to v2', () => { + expect(client['isVersionLTE']('1.40.0', '1.50.0')).equal(true, 'should be satisfied since v1 is less than v2'); + expect(client['isVersionLTE']('1.50.0', '1.50.0')).equal(true, 'should be satisfied since v1 and v2 are equal'); + expect(client['isVersionLTE']('2.0.2', '2.0.1')).equal(false, 'should not be satisfied since v1 is greater than v2'); + }); + + }); + +}); diff --git a/packages/vsx-registry/src/common/vsx-registry-api.ts b/dev-packages/ovsx-client/src/ovsx-client.ts similarity index 75% rename from packages/vsx-registry/src/common/vsx-registry-api.ts rename to dev-packages/ovsx-client/src/ovsx-client.ts index 945bd39442045..a9ba94225fcaa 100644 --- a/packages/vsx-registry/src/common/vsx-registry-api.ts +++ b/dev-packages/ovsx-client/src/ovsx-client.ts @@ -1,5 +1,5 @@ /******************************************************************************** - * Copyright (C) 2020 TypeFox and others. + * Copyright (C) 2021 TypeFox and others. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0 which is available at @@ -16,39 +16,31 @@ import * as bent from 'bent'; import * as semver from 'semver'; -import { injectable, inject } from '@theia/core/shared/inversify'; -import { VSXExtensionRaw, VSXSearchParam, VSXSearchResult, VSXAllVersions, VSXBuiltinNamespaces, VSXSearchEntry } from './vsx-registry-types'; -import { VSXEnvironment } from './vsx-environment'; -import { VSXApiVersionProvider } from './vsx-api-version-provider'; +import { + VSXAllVersions, + VSXBuiltinNamespaces, + VSXExtensionRaw, + VSXQueryParam, + VSXQueryResult, + VSXSearchEntry, + VSXSearchParam, + VSXSearchResult +} from './ovsx-types'; const fetchText = bent('GET', 'string', 200); -const fetchJson = bent('GET', { - 'Accept': 'application/json' -}, 'json', 200); +const fetchJson = bent('GET', { 'Accept': 'application/json' }, 'json', 200); const postJson = bent('POST', { 'Content-Type': 'application/json', 'Accept': 'application/json' }, 'json', 200); -export interface VSXResponseError extends Error { - statusCode: number +export interface OVSXClientOptions { + apiVersion: string + apiUrl: string } -export namespace VSXResponseError { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - export function is(error: any): error is VSXResponseError { - return !!error && typeof error === 'object' - && 'statusCode' in error && typeof error['statusCode'] === 'number'; - } -} - -@injectable() -export class VSXRegistryAPI { +export class OVSXClient { - @inject(VSXApiVersionProvider) - protected readonly apiVersionProvider: VSXApiVersionProvider; - - @inject(VSXEnvironment) - protected readonly environment: VSXEnvironment; + constructor(readonly options: OVSXClientOptions) { } async search(param?: VSXSearchParam): Promise { const searchUri = await this.buildSearchUri(param); @@ -56,8 +48,7 @@ export class VSXRegistryAPI { } protected async buildSearchUri(param?: VSXSearchParam): Promise { - const apiUri = await this.environment.getRegistryApiUri(); - let searchUri = apiUri.resolve('-/search').toString(); + let searchUri = ''; if (param) { const query: string[] = []; if (param.query) { @@ -85,15 +76,15 @@ export class VSXRegistryAPI { searchUri += '?' + query.join('&'); } } - return searchUri; + return new URL(`api/-/search${searchUri}`, this.options!.apiUrl).toString(); } async getExtension(id: string): Promise { - const apiUri = await this.environment.getRegistryApiUri(); - const param: QueryParam = { + const apiUri = new URL('api/-/query', this.options!.apiUrl); + const param: VSXQueryParam = { extensionId: id }; - const result = await this.postJson(apiUri.resolve('-/query').toString(), param); + const result = await this.postJson(apiUri.toString(), param); if (result.extensions && result.extensions.length > 0) { return result.extensions[0]; } @@ -105,12 +96,12 @@ export class VSXRegistryAPI { * @param id the requested extension id. */ async getAllVersions(id: string): Promise { - const apiUri = await this.environment.getRegistryApiUri(); - const param: QueryParam = { + const apiUri = new URL('api/-/query', this.options!.apiUrl); + const param: VSXQueryParam = { extensionId: id, includeAllVersions: true, }; - const result = await this.postJson(apiUri.resolve('-/query').toString(), param); + const result = await this.postJson(apiUri.toString(), param); if (result.extensions && result.extensions.length > 0) { return result.extensions; } @@ -145,7 +136,7 @@ export class VSXRegistryAPI { const namespace = extensions[0].namespace.toLowerCase(); if (this.isBuiltinNamespace(namespace)) { - const apiVersion = this.apiVersionProvider.getApiVersion(); + const apiVersion = this.options!.apiVersion; for (const extension of extensions) { if (this.isVersionLTE(extension.version, apiVersion)) { return extension; @@ -170,7 +161,7 @@ export class VSXRegistryAPI { getLatestCompatibleVersion(entry: VSXSearchEntry): VSXAllVersions | undefined { const extensions = entry.allVersions; if (this.isBuiltinNamespace(entry.namespace)) { - const apiVersion = this.apiVersionProvider.getApiVersion(); + const apiVersion = this.options!.apiVersion; for (const extension of extensions) { if (this.isVersionLTE(extension.version, apiVersion)) { return extension; @@ -200,8 +191,7 @@ export class VSXRegistryAPI { if (engine === '*') { return true; } else { - const apiVersion = this.apiVersionProvider.getApiVersion(); - return semver.satisfies(apiVersion, engine); + return semver.satisfies(this.options!.apiVersion, engine); } } @@ -230,17 +220,3 @@ export class VSXRegistryAPI { } } - -interface QueryParam { - namespaceName?: string; - extensionName?: string; - extensionVersion?: string; - extensionId?: string; - extensionUuid?: string; - namespaceUuid?: string; - includeAllVersions?: boolean; -} - -interface QueryResult { - extensions?: VSXExtensionRaw[]; -} diff --git a/packages/vsx-registry/src/common/vsx-registry-types.ts b/dev-packages/ovsx-client/src/ovsx-types.ts similarity index 87% rename from packages/vsx-registry/src/common/vsx-registry-types.ts rename to dev-packages/ovsx-client/src/ovsx-types.ts index d3f2ec13f823b..d536dd7897008 100644 --- a/packages/vsx-registry/src/common/vsx-registry-types.ts +++ b/dev-packages/ovsx-client/src/ovsx-types.ts @@ -144,6 +144,32 @@ export interface VSXExtensionRaw { readonly engines?: { [engine: string]: string }; } +export interface VSXQueryParam { + namespaceName?: string; + extensionName?: string; + extensionVersion?: string; + extensionId?: string; + extensionUuid?: string; + namespaceUuid?: string; + includeAllVersions?: boolean; +} + +export interface VSXQueryResult { + extensions?: VSXExtensionRaw[]; +} + +export interface VSXResponseError extends Error { + statusCode: number +} + +export namespace VSXResponseError { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + export function is(error: any): error is VSXResponseError { + return !!error && typeof error === 'object' + && 'statusCode' in error && typeof error['statusCode'] === 'number'; + } +} + /** * Builtin namespaces maintained by the framework. */ diff --git a/packages/plugin-ext-vscode/src/common/plugin-vscode-types.ts b/packages/plugin-ext-vscode/src/common/plugin-vscode-types.ts index c7d017f112a5a..4cf51272fadc0 100644 --- a/packages/plugin-ext-vscode/src/common/plugin-vscode-types.ts +++ b/packages/plugin-ext-vscode/src/common/plugin-vscode-types.ts @@ -15,3 +15,4 @@ ********************************************************************************/ export const VSCODE_DEFAULT_API_VERSION = '1.50.0'; +export const VSX_REGISTRY_URL_DEFAULT = 'https://open-vsx.org'; diff --git a/packages/vsx-registry/compile.tsconfig.json b/packages/vsx-registry/compile.tsconfig.json index 355e20446e63d..2d31e29bc79f4 100644 --- a/packages/vsx-registry/compile.tsconfig.json +++ b/packages/vsx-registry/compile.tsconfig.json @@ -20,6 +20,9 @@ }, { "path": "../filesystem/compile.tsconfig.json" + }, + { + "path": "../../dev-packages/ovsx-client/compile.tsconfig.json" } ] } diff --git a/packages/vsx-registry/package.json b/packages/vsx-registry/package.json index c89a5603d539a..be04b0c6e0ed5 100644 --- a/packages/vsx-registry/package.json +++ b/packages/vsx-registry/package.json @@ -5,13 +5,13 @@ "dependencies": { "@theia/core": "1.14.0", "@theia/filesystem": "1.14.0", + "@theia/ovsx-client": "1.14.0", "@theia/plugin-ext": "1.14.0", "@theia/plugin-ext-vscode": "1.14.0", "@types/bent": "^7.0.1", "@types/dompurify": "^2.0.2", "@types/sanitize-html": "^2.3.1", "@types/showdown": "^1.7.1", - "bent": "^7.1.0", "dompurify": "^2.0.11", "p-debounce": "^2.1.0", "requestretry": "^3.1.0", diff --git a/packages/vsx-registry/src/browser/ovsx-async-client.ts b/packages/vsx-registry/src/browser/ovsx-async-client.ts new file mode 100644 index 0000000000000..56871769e873d --- /dev/null +++ b/packages/vsx-registry/src/browser/ovsx-async-client.ts @@ -0,0 +1,39 @@ +/******************************************************************************** + * Copyright (C) 2021 Ericsson and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import { OVSXClient, OVSXClientOptions } from '@theia/ovsx-client/lib'; + +/** + * In some instances, the OVSXClient must be created asynchronously. This class + * makes it possible to get an un-initialized instance and wait for it to be ready. + */ +export class OVSXAsyncClient extends OVSXClient { + + /** + * Resolves once the initial asynchronous options are resolved. + * + * Calling methods before this promise is resolved will throw errors. + */ + readonly ready: Promise; + + constructor(asyncOptions: Promise) { + super(undefined!); // hack: using methods at this point will fail. + this.ready = asyncOptions.then(options => { + (this.options as OVSXClientOptions) = options; + return this; + }); + } +} diff --git a/packages/vsx-registry/src/browser/vsx-api-version-provider-frontend-impl.ts b/packages/vsx-registry/src/browser/vsx-api-version-provider-frontend-impl.ts deleted file mode 100644 index b18c4ac4b2e1f..0000000000000 --- a/packages/vsx-registry/src/browser/vsx-api-version-provider-frontend-impl.ts +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2020 Ericsson and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ - -import { inject, injectable } from '@theia/core/shared/inversify'; -import { VSXEnvironment } from '../common/vsx-environment'; -import { VSXApiVersionProvider } from '../common/vsx-api-version-provider'; -import { FrontendApplicationContribution, FrontendApplication } from '@theia/core/lib/browser'; - -@injectable() -export class VSXApiVersionProviderImpl implements VSXApiVersionProvider, FrontendApplicationContribution { - - @inject(VSXEnvironment) - protected readonly vsxEnvironment: VSXEnvironment; - - protected _apiVersion: string; - - async onStart(_app: FrontendApplication): Promise { - this._apiVersion = await this.vsxEnvironment.getVscodeApiVersion(); - } - - getApiVersion(): string { - return this._apiVersion; - } - -} diff --git a/packages/vsx-registry/src/browser/vsx-extension.tsx b/packages/vsx-registry/src/browser/vsx-extension.tsx index dfa669f7fa3a0..b64bf6093522f 100644 --- a/packages/vsx-registry/src/browser/vsx-extension.tsx +++ b/packages/vsx-registry/src/browser/vsx-extension.tsx @@ -27,9 +27,9 @@ import { ProgressService } from '@theia/core/lib/common/progress-service'; import { Endpoint } from '@theia/core/lib/browser/endpoint'; import { VSXEnvironment } from '../common/vsx-environment'; import { VSXExtensionsSearchModel } from './vsx-extensions-search-model'; -import { VSXExtensionNamespaceAccess, VSXUser } from '../common/vsx-registry-types'; import { MenuPath } from '@theia/core/lib/common'; import { ContextMenuRenderer } from '@theia/core/lib/browser'; +import { VSXExtensionNamespaceAccess, VSXUser } from '@theia/ovsx-client/lib/ovsx-types'; export const EXTENSIONS_CONTEXT_MENU: MenuPath = ['extensions_context_menu']; diff --git a/packages/vsx-registry/src/browser/vsx-extensions-model.ts b/packages/vsx-registry/src/browser/vsx-extensions-model.ts index a58229501a8f9..f0c1bc01b4284 100644 --- a/packages/vsx-registry/src/browser/vsx-extensions-model.ts +++ b/packages/vsx-registry/src/browser/vsx-extensions-model.ts @@ -20,13 +20,13 @@ import * as showdown from 'showdown'; import * as sanitize from 'sanitize-html'; import { Emitter } from '@theia/core/lib/common/event'; import { CancellationToken, CancellationTokenSource } from '@theia/core/lib/common/cancellation'; -import { VSXRegistryAPI, VSXResponseError } from '../common/vsx-registry-api'; -import { VSXSearchParam } from '../common/vsx-registry-types'; import { HostedPluginSupport } from '@theia/plugin-ext/lib/hosted/browser/hosted-plugin'; import { VSXExtension, VSXExtensionFactory } from './vsx-extension'; import { ProgressService } from '@theia/core/lib/common/progress-service'; import { VSXExtensionsSearchModel } from './vsx-extensions-search-model'; import { Deferred } from '@theia/core/lib/common/promise-util'; +import { VSXResponseError, VSXSearchParam } from '@theia/ovsx-client/lib/ovsx-types'; +import { OVSXAsyncClient } from './ovsx-async-client'; @injectable() export class VSXExtensionsModel { @@ -34,8 +34,8 @@ export class VSXExtensionsModel { protected readonly onDidChangeEmitter = new Emitter(); readonly onDidChange = this.onDidChangeEmitter.event; - @inject(VSXRegistryAPI) - protected readonly api: VSXRegistryAPI; + @inject(OVSXAsyncClient) + protected client: OVSXAsyncClient; @inject(HostedPluginSupport) protected readonly pluginSupport: HostedPluginSupport; @@ -53,6 +53,7 @@ export class VSXExtensionsModel { @postConstruct() protected async init(): Promise { + await this.client.ready; await Promise.all([ this.initInstalled(), this.initSearchResult() @@ -132,14 +133,14 @@ export class VSXExtensionsModel { }, 150); protected doUpdateSearchResult(param: VSXSearchParam, token: CancellationToken): Promise { return this.doChange(async () => { - const result = await this.api.search(param); + const result = await this.client.search(param); if (token.isCancellationRequested) { return; } const searchResult = new Set(); for (const data of result.extensions) { const id = data.namespace.toLowerCase() + '.' + data.name.toLowerCase(); - const extension = this.api.getLatestCompatibleVersion(data); + const extension = this.client.getLatestCompatibleVersion(data); if (!extension) { continue; } @@ -191,7 +192,7 @@ export class VSXExtensionsModel { } if (extension.readmeUrl) { try { - const rawReadme = await this.api.fetchText(extension.readmeUrl); + const rawReadme = await this.client.fetchText(extension.readmeUrl); const readme = this.compileReadme(rawReadme); extension.update({ readme }); } catch (e) { @@ -223,7 +224,7 @@ export class VSXExtensionsModel { if (!this.shouldRefresh(extension)) { return extension; } - const data = await this.api.getLatestCompatibleExtensionVersion(id); + const data = await this.client.getLatestCompatibleExtensionVersion(id); if (!data) { return; } diff --git a/packages/vsx-registry/src/browser/vsx-registry-frontend-module.ts b/packages/vsx-registry/src/browser/vsx-registry-frontend-module.ts index ec66b149d62d4..17f26ec281dd2 100644 --- a/packages/vsx-registry/src/browser/vsx-registry-frontend-module.ts +++ b/packages/vsx-registry/src/browser/vsx-registry-frontend-module.ts @@ -21,7 +21,6 @@ import { WidgetFactory, bindViewContribution, FrontendApplicationContribution, V import { VSXExtensionsViewContainer } from './vsx-extensions-view-container'; import { VSXExtensionsContribution } from './vsx-extensions-contribution'; import { VSXExtensionsSearchBar } from './vsx-extensions-search-bar'; -import { VSXRegistryAPI } from '../common/vsx-registry-api'; import { VSXExtensionsModel } from './vsx-extensions-model'; import { ColorContribution } from '@theia/core/lib/browser/color-application-contribution'; import { VSXExtensionsWidget, VSXExtensionsWidgetOptions } from './vsx-extensions-widget'; @@ -32,12 +31,20 @@ import { VSXExtensionEditorManager } from './vsx-extension-editor-manager'; import { VSXExtensionsSourceOptions } from './vsx-extensions-source'; import { VSXEnvironment } from '../common/vsx-environment'; import { VSXExtensionsSearchModel } from './vsx-extensions-search-model'; -import { VSXApiVersionProviderImpl } from './vsx-api-version-provider-frontend-impl'; -import { VSXApiVersionProvider } from '../common/vsx-api-version-provider'; +import { OVSXAsyncClient } from './ovsx-async-client'; export default new ContainerModule(bind => { - bind(VSXEnvironment).toSelf().inRequestScope(); - bind(VSXRegistryAPI).toSelf().inSingletonScope(); + bind(VSXEnvironment).toSelf().inSingletonScope(); + bind(OVSXAsyncClient).toDynamicValue(ctx => { + const vsxEnvironment = ctx.container.get(VSXEnvironment); + return new OVSXAsyncClient(Promise.all([ + vsxEnvironment.getVscodeApiVersion(), + vsxEnvironment.getRegistryApiUri() + ]).then(([apiVersion, apiUri]) => ({ + apiVersion, + apiUrl: apiUri.toString() + }))); + }).inSingletonScope(); bind(VSXExtension).toSelf(); bind(VSXExtensionFactory).toFactory(ctx => (option: VSXExtensionOptions) => { @@ -92,8 +99,4 @@ export default new ContainerModule(bind => { bind(FrontendApplicationContribution).toService(VSXExtensionsContribution); bind(ColorContribution).toService(VSXExtensionsContribution); bind(TabBarToolbarContribution).toService(VSXExtensionsContribution); - - bind(VSXApiVersionProviderImpl).toSelf().inSingletonScope(); - bind(FrontendApplicationContribution).toService(VSXApiVersionProviderImpl); - bind(VSXApiVersionProvider).toService(VSXApiVersionProviderImpl); }); diff --git a/packages/vsx-registry/src/common/vsx-registry-api.spec.ts b/packages/vsx-registry/src/common/vsx-registry-api.spec.ts deleted file mode 100644 index 0a95c7c5b2f01..0000000000000 --- a/packages/vsx-registry/src/common/vsx-registry-api.spec.ts +++ /dev/null @@ -1,168 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2020 Ericsson and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ - -import * as chai from 'chai'; -import URI from '@theia/core/lib/common/uri'; -import { Container } from '@theia/core/shared/inversify'; -import { VSXEnvironment } from './vsx-environment'; -import { VSXRegistryAPI } from './vsx-registry-api'; -import { VSXSearchParam } from './vsx-registry-types'; -import { VSXApiVersionProvider } from './vsx-api-version-provider'; -import { EnvVariablesServer } from '@theia/core/lib/common/env-variables/env-variables-protocol'; - -const expect = chai.expect; - -describe('VSX Registry API', () => { - - let api: VSXRegistryAPI; - - beforeEach(() => { - const container = new Container(); - container.bind(VSXRegistryAPI).toSelf().inSingletonScope(); - container.bind(VSXEnvironment).toConstantValue({ - async getRegistryApiUri(): Promise { - return new URI('https://open-vsx.org/api'); - }, - async getRegistryUri(): Promise { - return new URI('https://open-vsx.org'); - }, - async getVscodeApiVersion(): Promise { - return '1.40.0'; - } - }); - container.bind(EnvVariablesServer).toConstantValue({}); - container.bind(VSXApiVersionProvider).toConstantValue({ - getApiVersion(): string { - return '1.40.0'; - } - }); - api = container.get(VSXRegistryAPI); - }); - - describe('isEngineValid', () => { - - it('should return \'true\' for a compatible engine', () => { - const a: boolean = api['isEngineSupported']('^1.20.0'); - const b: boolean = api['isEngineSupported']('^1.40.0'); - expect(a).to.eq(true); - expect(b).to.eq(true); - }); - - it('should return \'false\' for a incompatible engine', () => { - const valid: boolean = api['isEngineSupported']('^1.50.0'); - expect(valid).to.eq(false); - }); - - it('should return \'false\' for an undefined engine', () => { - const valid: boolean = api['isEngineSupported'](); - expect(valid).to.eq(false); - }); - - }); - - describe('#buildSearchUri', () => { - - it('should correctly build the search URI with the single `query` parameter present', async () => { - const expected = 'https://open-vsx.org/api/-/search?query=javascript'; - const param: VSXSearchParam = { - query: 'javascript', - }; - const query = await api['buildSearchUri'](param); - expect(query).to.eq(expected); - }); - - it('should correctly build the search URI with the multiple search parameters present', async () => { - let expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&includeAllVersions=true'; - let param: VSXSearchParam = { - query: 'javascript', - category: 'languages', - size: 20, - offset: 10, - includeAllVersions: true, - }; - let query = await api['buildSearchUri'](param); - expect(query).to.eq(expected); - - expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&sortOrder=desc&sortBy=relevance&includeAllVersions=true'; - param = { - query: 'javascript', - category: 'languages', - size: 20, - offset: 10, - sortOrder: 'desc', - sortBy: 'relevance', - includeAllVersions: true - }; - query = await api['buildSearchUri'](param); - expect(query).to.eq(expected); - }); - - }); - - describe('#buildSearchUri', () => { - - it('should correctly build the search URI with the single `query` parameter present', async () => { - - it('should build a proper query with the single `query` parameter present', async () => { - const expected = 'https://open-vsx.org/api/-/search?query=javascript'; - const param: VSXSearchParam = { - query: 'javascript', - }; - const query = await api['buildSearchUri'](param); - expect(query).to.eq(expected); - }); - - it('should correctly build the search URI with the multiple search parameters present', async () => { - let expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&includeAllVersions=true'; - let param: VSXSearchParam = { - query: 'javascript', - category: 'languages', - size: 20, - offset: 10, - includeAllVersions: true, - }; - let query = await api['buildSearchUri'](param); - expect(query).to.eq(expected); - - expected = 'https://open-vsx.org/api/-/search?query=javascript&category=languages&size=20&offset=10&sortOrder=desc&sortBy=relevance&includeAllVersions=true'; - param = { - query: 'javascript', - category: 'languages', - size: 20, - offset: 10, - sortOrder: 'desc', - sortBy: 'relevance', - includeAllVersions: true - }; - query = await api['buildSearchUri'](param); - expect(query).to.eq(expected); - }); - - }); - - }); - - describe('#isVersionLTE', () => { - - it('should determine if v1 is less than or equal to v2', () => { - expect(api['isVersionLTE']('1.40.0', '1.50.0')).equal(true, 'should be satisfied since v1 is less than v2'); - expect(api['isVersionLTE']('1.50.0', '1.50.0')).equal(true, 'should be satisfied since v1 and v2 are equal'); - expect(api['isVersionLTE']('2.0.2', '2.0.1')).equal(false, 'should not be satisfied since v1 is greater than v2'); - }); - - }); - -}); diff --git a/packages/vsx-registry/src/node/vsx-api-version-provider-backend-impl.ts b/packages/vsx-registry/src/node/vsx-api-version-provider-backend-impl.ts deleted file mode 100644 index 8ad03d239fbcb..0000000000000 --- a/packages/vsx-registry/src/node/vsx-api-version-provider-backend-impl.ts +++ /dev/null @@ -1,28 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2020 Ericsson and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ - -import { injectable } from '@theia/core/shared/inversify'; -import { VSXApiVersionProvider } from '../common/vsx-api-version-provider'; -import { VSCODE_DEFAULT_API_VERSION } from '@theia/plugin-ext-vscode/lib/common/plugin-vscode-types'; - -@injectable() -export class VSXApiVersionProviderImpl implements VSXApiVersionProvider { - - getApiVersion(): string { - return process.env['VSCODE_API_VERSION'] || VSCODE_DEFAULT_API_VERSION; - } - -} diff --git a/packages/vsx-registry/src/node/vsx-extension-resolver.ts b/packages/vsx-registry/src/node/vsx-extension-resolver.ts index 0152081892159..029afe1470078 100644 --- a/packages/vsx-registry/src/node/vsx-extension-resolver.ts +++ b/packages/vsx-registry/src/node/vsx-extension-resolver.ts @@ -23,13 +23,13 @@ import { injectable, inject } from '@theia/core/shared/inversify'; import URI from '@theia/core/lib/common/uri'; import { PluginDeployerResolver, PluginDeployerResolverContext } from '@theia/plugin-ext/lib/common/plugin-protocol'; import { VSXExtensionUri } from '../common/vsx-extension-uri'; -import { VSXRegistryAPI } from '../common/vsx-registry-api'; +import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client'; @injectable() export class VSXExtensionResolver implements PluginDeployerResolver { - @inject(VSXRegistryAPI) - protected readonly api: VSXRegistryAPI; + @inject(OVSXClient) + protected client: OVSXClient; protected readonly downloadPath: string; @@ -49,7 +49,7 @@ export class VSXExtensionResolver implements PluginDeployerResolver { return; } console.log(`[${id}]: trying to resolve latest version...`); - const extension = await this.api.getLatestCompatibleExtensionVersion(id); + const extension = await this.client.getLatestCompatibleExtensionVersion(id); if (!extension) { return; } diff --git a/packages/vsx-registry/src/node/vsx-registry-backend-module.ts b/packages/vsx-registry/src/node/vsx-registry-backend-module.ts index 5738090d33144..cbb4a41f767f5 100644 --- a/packages/vsx-registry/src/node/vsx-registry-backend-module.ts +++ b/packages/vsx-registry/src/node/vsx-registry-backend-module.ts @@ -17,18 +17,18 @@ import { ContainerModule } from '@theia/core/shared/inversify'; import { VSXExtensionResolver } from './vsx-extension-resolver'; import { PluginDeployerResolver } from '@theia/plugin-ext/lib/common/plugin-protocol'; -import { VSXRegistryAPI } from '../common/vsx-registry-api'; -import { VSXEnvironment } from '../common/vsx-environment'; -import { VSXApiVersionProviderImpl } from './vsx-api-version-provider-backend-impl'; -import { VSXApiVersionProvider } from '../common/vsx-api-version-provider'; +import { VSCODE_DEFAULT_API_VERSION, VSX_REGISTRY_URL_DEFAULT } from '@theia/plugin-ext-vscode/lib/common/plugin-vscode-types'; +import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client'; export default new ContainerModule(bind => { - bind(VSXEnvironment).toSelf().inRequestScope(); - bind(VSXRegistryAPI).toSelf().inSingletonScope(); - + bind(OVSXClient).toConstantValue(new OVSXClient({ + apiVersion: process.env['VSCODE_API_VERSION'] || VSCODE_DEFAULT_API_VERSION, + apiUrl: resolveRegistryUrl() + })); bind(VSXExtensionResolver).toSelf().inSingletonScope(); bind(PluginDeployerResolver).toService(VSXExtensionResolver); - - bind(VSXApiVersionProviderImpl).toSelf().inSingletonScope(); - bind(VSXApiVersionProvider).toService(VSXApiVersionProviderImpl); }); + +function resolveRegistryUrl(): string { + return process.env['VSX_REGISTRY_URL'] || VSX_REGISTRY_URL_DEFAULT; +} diff --git a/tsconfig.json b/tsconfig.json index 6dc07ada89dc2..3bf2314107214 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -180,6 +180,9 @@ ], "@theia/eslint-plugin/*": [ "dev-packages/eslint-plugin/*" + ], + "@theia/ovsx-client/lib/*": [ + "dev-packages/ovsx-client/src/*" ] } }