Support http proxies

dev-packages/cli/package.json

@@ -35,17 +35,15 @@
     "@theia/ffmpeg": "1.25.0",
     "@theia/localization-manager": "1.25.0",
     "@theia/ovsx-client": "1.25.0",
+    "@theia/request": "1.25.0",
     "@types/chai": "^4.2.7",
     "@types/mocha": "^5.2.7",
     "@types/node-fetch": "^2.5.7",
     "@types/puppeteer": "^2.0.0",
     "chai": "^4.2.0",
     "chalk": "4.0.0",
     "decompress": "^4.2.1",
-    "https-proxy-agent": "^5.0.0",
     "mocha": "^7.0.0",
-    "node-fetch": "^2.6.7",
-    "proxy-from-env": "^1.1.0",
     "puppeteer": "^2.0.0",
     "puppeteer-to-istanbul": "^1.2.2",
     "temp": "^0.9.1",

dev-packages/cli/src/download-plugins.ts

@@ -26,17 +26,12 @@ declare global {
 import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
 import * as chalk from 'chalk';
 import * as decompress from 'decompress';
-import { createWriteStream, promises as fs } from 'fs';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import fetch, { RequestInit, Response } from 'node-fetch';
+import { promises as fs } from 'fs';
 import * as path from 'path';
-import { getProxyForUrl } from 'proxy-from-env';
-import * as stream from 'stream';
 import * as temp from 'temp';
-import { promisify } from 'util';
+import { NodeRequestService } from '@theia/request/lib/node-request-service';
 import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';
-
-const pipelineAsPromised = promisify(stream.pipeline);
+import { RequestContext } from '@theia/request';
 
 temp.track();
 
@@ -71,6 +66,10 @@ export interface DownloadPluginsOptions {
      * Fetch plugins in parallel
      */
     parallel?: boolean;
+
+    proxyUrl?: string;
+    proxyAuthorization?: string;
+    strictSsl?: boolean;
 }
 
 interface PluginDownload {
@@ -79,15 +78,26 @@ interface PluginDownload {
     version?: string | undefined
 }
 
+const requestService = new NodeRequestService();
+
 export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise<void> {
     const {
         packed = false,
         ignoreErrors = false,
         apiVersion = DEFAULT_SUPPORTED_API_VERSION,
         apiUrl = 'https://open-vsx.org/api',
-        parallel = true
+        parallel = true,
+        proxyUrl,
+        proxyAuthorization,
+        strictSsl
     } = options;
 
+    requestService.configure({
+        proxyUrl,
+        proxyAuthorization,
+        strictSSL: strictSsl
+    });
+
     // Collect the list of failures to be appended at the end of the script.
     const failures: string[] = [];
 
@@ -133,7 +143,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         await downloader(pluginsToDownload);
 
         const handleDependencyList = async (dependencies: Array<string | string[]>) => {
-            const client = new OVSXClient({ apiVersion, apiUrl });
+            const client = new OVSXClient({ apiVersion, apiUrl }, requestService);
             // De-duplicate extension ids to only download each once:
             const ids = new Set<string>(dependencies.flat());
             await parallelOrSequence(...Array.from(ids, id => async () => {
@@ -210,20 +220,23 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
 
     let attempts: number;
     let lastError: Error | undefined;
-    let response: Response | undefined;
+    let response: RequestContext | undefined;
 
     for (attempts = 0; attempts < maxAttempts; attempts++) {
         if (attempts > 0) {
             await new Promise(resolve => setTimeout(resolve, retryDelay));
         }
         lastError = undefined;
         try {
-            response = await xfetch(pluginUrl);
+            response = await requestService.request({
+                url: pluginUrl
+            });
         } catch (error) {
             lastError = error;
             continue;
         }
-        const retry = response.status === 439 || response.status >= 500;
+        const status = response.res.statusCode;
+        const retry = status && (status === 439 || status >= 500);
         if (!retry) {
             break;
         }
@@ -236,20 +249,19 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         failures.push(chalk.red(`x ${plugin}: failed to download (unknown reason)`));
         return;
     }
-    if (response.status !== 200) {
-        failures.push(chalk.red(`x ${plugin}: failed to download with: ${response.status} ${response.statusText}`));
+    if (response.res.statusCode !== 200) {
+        failures.push(chalk.red(`x ${plugin}: failed to download with: ${response.res.statusCode}`));
         return;
     }
 
     if ((fileExt === '.vsix' || fileExt === '.theia') && packed === true) {
         // Download .vsix without decompressing.
-        const file = createWriteStream(targetPath);
-        await pipelineAsPromised(response.body, file);
+        await fs.writeFile(targetPath, response.buffer);
     } else {
         await fs.mkdir(targetPath, { recursive: true });
-        const tempFile = temp.createWriteStream('theia-plugin-download');
-        await pipelineAsPromised(response.body, tempFile);
-        await decompress(tempFile.path, targetPath);
+        const tempFile = temp.path('theia-plugin-download');
+        await fs.writeFile(tempFile, response.buffer);
+        await decompress(tempFile, targetPath);
     }
 
     console.warn(chalk.green(`+ ${plugin}${version ? `@${version}` : ''}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
@@ -265,18 +277,6 @@ async function isDownloaded(filePath: string): Promise<boolean> {
     return fs.stat(filePath).then(() => true, () => false);
 }
 
-/**
- * Follow HTTP(S)_PROXY, ALL_PROXY and NO_PROXY environment variables.
- */
-export function xfetch(url: string, options?: RequestInit): Promise<Response> {
-    const proxiedOptions: RequestInit = { ...options };
-    const proxy = getProxyForUrl(url);
-    if (!proxiedOptions.agent && proxy !== '') {
-        proxiedOptions.agent = new HttpsProxyAgent(proxy);
-    }
-    return fetch(url, proxiedOptions);
-}
-
 /**
  * Walk the plugin directory and collect available extension paths.
  * @param pluginDir the plugin directory.

dev-packages/cli/src/theia.ts

@@ -197,11 +197,14 @@ async function theiaCli(): Promise<void> {
             }
         })
         .command<{
-            packed: boolean,
-            ignoreErrors: boolean,
-            apiVersion: string,
-            apiUrl: string,
+            packed: boolean
+            ignoreErrors: boolean
+            apiVersion: string
+            apiUrl: string
             parallel: boolean
+            proxyUrl?: string
+            proxyAuthentification?: string
+            strictSsl: boolean
         }>({
             command: 'download:plugins',
             describe: 'Download defined external plugins',
@@ -232,10 +235,21 @@ async function theiaCli(): Promise<void> {
                     describe: 'Download in parallel',
                     boolean: true,
                     default: true
+                },
+                'proxy-url': {
+                    describe: 'Proxy URL'
+                },
+                'proxy-authentification': {
+                    describe: 'Proxy authentification information'
+                },
+                'strict-ssl': {
+                    describe: 'Whether to enable strict SSL mode',
+                    boolean: true,
+                    default: false
                 }
             },
-            handler: async ({ packed, ignoreErrors, apiVersion, apiUrl, parallel }) => {
-                await downloadPlugins({ packed, ignoreErrors, apiVersion, apiUrl, parallel });
+            handler: async args => {
+                await downloadPlugins(args);
             },
         })
         .command<{

dev-packages/cli/tsconfig.json

@@ -23,6 +23,9 @@
     },
     {
       "path": "../ovsx-client"
+    },
+    {
+      "path": "../request"
     }
   ]
 }

dev-packages/ovsx-client/package.json

@@ -29,8 +29,7 @@
     "watch": "theiaext watch"
   },
   "dependencies": {
-    "@types/bent": "^7.0.1",
-    "bent": "^7.1.0",
+    "@theia/request": "1.25.0",
     "semver": "^5.4.1"
   }
 }

dev-packages/ovsx-client/src/ovsx-client.spec.ts

@@ -16,6 +16,7 @@
 
 import * as chai from 'chai';
 import { OVSXClient } from './ovsx-client';
+import { NodeRequestService } from '@theia/request/lib/node-request-service';
 import { VSXSearchParam } from './ovsx-types';
 
 const expect = chai.expect;
@@ -31,7 +32,7 @@ describe('OVSX Client', () => {
         client = new OVSXClient({
             apiVersion,
             apiUrl
-        });
+        }, new NodeRequestService());
     });
 
     describe('isEngineValid', () => {

dev-packages/ovsx-client/src/ovsx-client.ts

@@ -14,7 +14,6 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import * as bent from 'bent';
 import * as semver from 'semver';
 import {
     VSXAllVersions,
@@ -26,9 +25,7 @@ import {
     VSXSearchParam,
     VSXSearchResult
 } from './ovsx-types';
-
-const fetchText = bent('GET', 'string', 200);
-const fetchJson = bent('GET', { 'Accept': 'application/json' }, 'json', 200);
+import { RequestContext, RequestService } from '@theia/request';
 
 export interface OVSXClientOptions {
     apiVersion: string
@@ -37,11 +34,19 @@ export interface OVSXClientOptions {
 
 export class OVSXClient {
 
-    constructor(readonly options: OVSXClientOptions) { }
+    constructor(readonly options: OVSXClientOptions, protected readonly request: RequestService) { }
 
     async search(param?: VSXSearchParam): Promise<VSXSearchResult> {
         const searchUri = await this.buildSearchUri(param);
-        return this.fetchJson<VSXSearchResult>(searchUri);
+        try {
+            return await this.fetchJson<VSXSearchResult>(searchUri);
+        } catch (err) {
+            return {
+                error: err?.message || String(err),
+                offset: 0,
+                extensions: []
+            };
+        }
     }
 
     protected async buildSearchUri(param?: VSXSearchParam): Promise<string> {
@@ -99,12 +104,17 @@ export class OVSXClient {
         throw new Error(`Extension with id ${id} not found at ${apiUri}`);
     }
 
-    protected fetchJson<R>(url: string): Promise<R> {
-        return fetchJson(url) as Promise<R>;
+    protected async fetchJson<R>(url: string): Promise<R> {
+        const requestContext = await this.request.request({
+            url,
+            headers: { 'Accept': 'application/json' }
+        });
+        return RequestContext.asJson<R>(requestContext);
     }
 
-    fetchText(url: string): Promise<string> {
-        return fetchText(url);
+    async fetchText(url: string): Promise<string> {
+        const requestContext = await this.request.request({ url });
+        return RequestContext.asText(requestContext);
     }
 
     /**

dev-packages/ovsx-client/tsconfig.json

@@ -8,5 +8,9 @@
   "include": [
     "src"
   ],
-  "references": []
+  "references": [
+    {
+      "path": "../request"
+    }
+  ]
 }

dev-packages/request/.eslintrc.js

@@ -0,0 +1,10 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+    extends: [
+        '../../configs/build.eslintrc.json'
+    ],
+    parserOptions: {
+        tsconfigRootDir: __dirname,
+        project: 'tsconfig.json'
+    }
+};

dev-packages/request/README.md

@@ -0,0 +1,29 @@
+<div align='center'>
+
+<br />
+
+<img src='https://raw.githubusercontent.com/eclipse-theia/theia/master/logo/theia.svg?sanitize=true' alt='theia-ext-logo' width='100px' />
+
+<h2>ECLIPSE THEIA - REQUEST</h2>
+
+<hr />
+
+</div>
+
+## Description
+
+The `@theia/request` package is used to send proxy-aware http requests to other services.
+
+## Additional Information
+
+- [Theia - GitHub](https://github.com/eclipse-theia/theia)
+- [Theia - Website](https://theia-ide.org/)
+
+## License
+
+- [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/)
+- [一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp)
+
+## Trademark
+"Theia" is a trademark of the Eclipse Foundation
+https://www.eclipse.org/theia

dev-packages/request/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@theia/request",
+  "version": "1.25.0",
+  "description": "Theia Proxy-Aware Request Service",
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/eclipse-theia/theia.git"
+  },
+  "bugs": {
+    "url": "https://github.com/eclipse-theia/theia/issues"
+  },
+  "homepage": "https://github.com/eclipse-theia/theia",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "scripts": {
+    "build": "theiaext build",
+    "clean": "theiaext clean",
+    "compile": "theiaext compile",
+    "lint": "theiaext lint",
+    "test": "theiaext test",
+    "watch": "theiaext watch"
+  },
+  "dependencies": {
+    "http-proxy-agent": "^5.0.0",
+    "https-proxy-agent": "^5.0.0"
+  }
+}