Proxy support (#674)

* Proxy support

* remove unused and lint

* log teardown

* skip tb

* node 14

* remove skip and node update

* remove consle

* Remove stray newline after merge

* upgrade deps

* update locl

* Avoid upsetting the linter

* detectOpenHandles

* remove jest global

* max tiemout

* --testTimeout param

* tb back

* tb partial tests

* tb partial tests 2

* skip tb test

* tb skip change

* isolating tb test

* isolating tb test

* commit one

* merge master [skip ci]

* no proxy

* no timeout

* Run tests with global timeout

Co-authored-by: Casper da Costa-Luis <casper.dcl@physics.org>

* remove skip

* forceexit

Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com>
Co-authored-by: Casper da Costa-Luis <casper.dcl@physics.org>

README.md

@@ -554,6 +554,11 @@ required.
 For AWS, the same credentials can also be used for
 [configuring cloud storage](#configuring-cloud-storage-providers).
 
+#### Proxy support
+
+CML support proxy via known environment variables `http_proxy` and
+`https_proxy`.
+
 #### On-premise (Local) Runners
 
 This means using on-premise machines as self-hosted runners. The `cml-runner`

package.json

@@ -42,7 +42,7 @@
   "scripts": {
     "lintfix": "eslint --fix ./ && prettier --write '**/*.{js,json,md,yaml,yml}'",
     "lint": "eslint ./",
-    "test": "jest",
+    "test": "jest --forceExit",
     "do_snapshots": "jest --updateSnapshot"
   },
   "husky": {
@@ -69,6 +69,7 @@
     "fs-extra": "^9.1.0",
     "git-url-parse": "^11.6.0",
     "globby": "^11.0.4",
+    "https-proxy-agent": "^5.0.0",
     "js-base64": "^3.7.0",
     "mmmagic": "^0.5.3",
     "node-fetch": "^2.6.2",
@@ -96,11 +97,14 @@
     "husky": "^4.3.8",
     "jest": "^27.1.1",
     "lint-staged": "^10.5.4",
-    "prettier": "^2.4.0"
+    "prettier": "^2.4.0",
+    "transparent-proxy": "^1.8.5"
   },
   "description": "<p align=\"center\">   <img src=\"https://static.iterative.ai/img/cml/title_strip_trim.png\" width=400> </p>",
   "homepage": "https://github.com/iterative/cml#readme",
   "jest": {
+    "globalSetup": "./tests/setup.js",
+    "globalTeardown": "./tests/teardown.js",
     "testTimeout": 600000
   }
 }

src/cml.test.js

@@ -1,4 +1,5 @@
 const CML = require('../src/cml').default;
+
 describe('Github tests', () => {
   const OLD_ENV = process.env;
 

src/drivers/bitbucket_cloud.js

@@ -1,6 +1,8 @@
 const fetch = require('node-fetch');
 const { URL } = require('url');
 
+const { proxyAgent } = require('../utils');
+
 const { BITBUCKET_COMMIT, BITBUCKET_BRANCH } = process.env;
 class BitbucketCloud {
   constructor(opts = {}) {
@@ -181,10 +183,12 @@ class BitbucketCloud {
       'Content-Type': 'application/json',
       Authorization: 'Basic ' + `${token}`
     };
+
     const response = await fetch(url || `${api}${endpoint}`, {
       method,
       headers,
-      body
+      body,
+      agent: proxyAgent()
     });
 
     if (response.status > 300) {

src/drivers/bitbucket_cloud.test.js

@@ -4,29 +4,35 @@ const {
   TEST_BBCLOUD_REPO: REPO,
   TEST_BBCLOUD_SHA: SHA
 } = process.env;
+
 describe('Non Enviromental tests', () => {
   const client = new BitbucketCloud({ repo: REPO, token: TOKEN });
+
   test('test repo and token', async () => {
     expect(client.repo).toBe(REPO);
     expect(client.token).toBe(TOKEN);
   });
+
   test('Comment', async () => {
     const report = '## Test comment';
     const commitSha = SHA;
 
     await client.commentCreate({ report, commitSha });
   });
+
   test('Check', async () => {
     await expect(client.checkCreate()).rejects.toThrow(
       'Bitbucket Cloud does not support check!'
     );
   });
+
   test('Publish', async () => {
     const path = `${__dirname}/../../assets/logo.png`;
     await expect(client.upload({ path })).rejects.toThrow(
       'Bitbucket Cloud does not support upload!'
     );
   });
+
   test('Runner token', async () => {
     await expect(client.runnerToken()).rejects.toThrow(
       'Bitbucket Cloud does not support runnerToken!'

src/drivers/github.js

@@ -9,8 +9,8 @@ const { Octokit } = require('@octokit/rest');
 const { throttling } = require('@octokit/plugin-throttling');
 const tar = require('tar');
 
+const { download, exec, proxyAgent } = require('../utils');
 const winston = require('winston');
-const { download, exec } = require('../utils');
 
 const CHECK_TITLE = 'CML Report';
 process.env.RUNNER_ALLOW_RUNASROOT = 1;
@@ -53,6 +53,7 @@ const octokit = (token, repo) => {
     }
   };
   const octokitOptions = {
+    request: { agent: proxyAgent() },
     auth: token,
     throttle: {
       onRateLimit: throttleHandler,

src/drivers/gitlab.js

@@ -6,7 +6,7 @@ const fs = require('fs').promises;
 const fse = require('fs-extra');
 const { resolve } = require('path');
 
-const { fetchUploadData, download, exec } = require('../utils');
+const { fetchUploadData, download, exec, proxyAgent } = require('../utils');
 
 const {
   IN_DOCKER,
@@ -275,7 +275,12 @@ class Gitlab {
     if (!url) throw new Error('Gitlab API endpoint not found');
 
     const headers = { 'PRIVATE-TOKEN': token, Accept: 'application/json' };
-    const response = await fetch(url, { method, headers, body });
+    const response = await fetch(url, {
+      method,
+      headers,
+      body,
+      agent: proxyAgent()
+    });
 
     if (response.status > 300) throw new Error(response.statusText);
     if (raw) return response;

src/utils.js

@@ -1,4 +1,5 @@
 const fetch = require('node-fetch');
+const HttpsProxyAgent = require('https-proxy-agent');
 const fs = require('fs');
 const PATH = require('path');
 const mmm = require('mmmagic');
@@ -165,6 +166,14 @@ const sshConnection = async (opts) => {
   return ssh;
 };
 
+const proxyAgent = (opts = {}) => {
+  const { https_proxy: httpsProxy, http_proxy: httpProxy } = process.env;
+  const { url = httpsProxy || httpProxy } = opts;
+
+  if (!url) return;
+  return new HttpsProxyAgent(url);
+};
+
 exports.exec = exec;
 exports.fetchUploadData = fetchUploadData;
 exports.upload = upload;
@@ -175,3 +184,4 @@ exports.sshPublicFromPrivateRsa = sshPublicFromPrivateRsa;
 exports.watermarkUri = watermarkUri;
 exports.download = download;
 exports.sshConnection = sshConnection;
+exports.proxyAgent = proxyAgent;

src/utils.test.js

@@ -1,4 +1,9 @@
-const { exec, upload, sshPublicFromPrivateRsa } = require('./utils');
+const {
+  exec,
+  upload,
+  sshPublicFromPrivateRsa,
+  proxyAgent
+} = require('./utils');
 
 describe('exec tests', () => {
   test('exec is await and outputs hello', async () => {
@@ -44,4 +49,23 @@ describe('Other tests', () => {
       'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwMFO1qTq68Uao1TU6W+911SWZN+mce+OTodS5mne0vMiWm7AFJuaxMKzL1NGeXOAUY+oeTAE7NCyi56mqEo3R8kOvwYbIQKmz27lzL0ZniI9UkYkE9aFslvTgou6cqcJO6S8GMNnJLLXbmX37zGUgZyuTAuot885WHqKyW0Pg/zplBNIPBRA+yhQG8z17dj8VwixoE7KgRiNK9CN5Yz/2TRlsHLan9LH9vHSK477bj+jPtbII4V5ZP5Du/70Seb1fe3648DVpg4mjDmLrQkW/rtNanKMVLw3bD9VroeLe3PW91nLq+noOuljPETtxxrhGagE28U6igPTmIQYczrpz ';
     expect(publicKey).toBe(expected);
   });
+
+  describe('proxyAgent', () => {
+    const { http_proxy: httpProxy, https_proxy: httpsProxy } = process.env;
+    process.env.http_proxy = 'https://www.example1.com';
+    expect(proxyAgent().proxy.host).toBe('www.example1.com');
+
+    // https_proxy overrides http_proxy
+    process.env.https_proxy = 'https://www.example2.com';
+    expect(proxyAgent().proxy.host).toBe('www.example2.com');
+
+    const url = 'https://www.example.com';
+    expect(proxyAgent({ url: `${url}` }).proxy.host).toBe('www.example.com');
+
+    process.env = {
+      ...process.env,
+      http_proxy: httpProxy,
+      https_proxy: httpsProxy
+    };
+  });
 });

tests/proxy.js

@@ -0,0 +1,22 @@
+const ProxyServer = require('transparent-proxy');
+
+const PORT = 9093;
+const startProxy = () => {
+  if (global.proxyTestsServer) return;
+
+  global.proxyTestsServer = new ProxyServer();
+  global.proxyTestsServer.listen(PORT, '0.0.0.0', () =>
+    console.log(`Proxy listening on port ${PORT}`)
+  );
+};
+const stopProxy = () => {
+  console.log('Teardown Jest. Stoping Proxy...');
+  global.proxyTestsServer.close();
+  global.proxyTestsServer.unref();
+};
+
+module.exports = {
+  startProxy,
+  stopProxy,
+  PORT
+};

tests/setup.js

@@ -0,0 +1,5 @@
+const { startProxy, PORT } = require('./proxy');
+module.exports = () => {
+  process.env.http_proxy = `http://localhost:${PORT}`;
+  startProxy();
+};

tests/teardown.js

@@ -0,0 +1,4 @@
+const { stopProxy } = require('./proxy');
+module.exports = () => {
+  stopProxy();
+};