Add HTTP(S) over HTTP(S) proxy support (#322)

* Add proxy support using tunnel package

# Conflicts:
#	lib/axiosHttpClient.ts
#	lib/policies/proxyPolicy.ts
#	lib/serviceClient.ts
#	package.json

* Fix incorrect merge

* Add tests

* Remove commented code

* Add tunnel to rollup configuration

* Fix test title casing

* Remove only

* Add axios client tests

* Mock buffer

* Remove rewire

* Fix default HTTP client tests

* Add some proxy tests

* Add support for HTTPS proxy

* Address PR comments

lib/axiosHttpClient.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
-import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, AxiosProxyConfig } from "axios";
+import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
 import { Transform, Readable } from "stream";
 import FormData from "form-data";
 import * as tough from "tough-cookie";
@@ -10,9 +10,11 @@ import { HttpHeaders } from "./httpHeaders";
 import { HttpOperationResponse } from "./httpOperationResponse";
 import { RestError } from "./restError";
 import { WebResource, HttpRequestBody } from "./webResource";
+import * as tunnel from "tunnel";
 import { ProxySettings } from "./serviceClient";
-
-export const axiosClient = axios.create();
+import http from "http";
+import https from "https";
+import { URLBuilder } from "./url";
 
 /**
  * A HttpClient implementation that uses axios to send HTTP requests.
@@ -130,9 +132,19 @@ export class AxiosHttpClient implements HttpClient {
         responseType: httpRequest.streamResponseBody ? "stream" : "text",
         cancelToken,
         timeout: httpRequest.timeout,
-        proxy: convertToAxiosProxyConfig(httpRequest.proxySettings)
+        proxy: false
       };
-      res = await axiosClient(config);
+
+      if (httpRequest.proxySettings) {
+        const agent = createProxyAgent(httpRequest.url, httpRequest.proxySettings, httpRequest.headers);
+        if (agent.isHttps) {
+          config.httpsAgent = agent.agent;
+        } else {
+          config.httpAgent = agent.agent;
+        }
+      }
+
+      res = await axios.request(config);
     } catch (err) {
       if (err instanceof axios.Cancel) {
         throw new RestError(err.message, RestError.REQUEST_SEND_ERROR, undefined, httpRequest);
@@ -198,25 +210,45 @@ export class AxiosHttpClient implements HttpClient {
   }
 }
 
-function convertToAxiosProxyConfig(proxySettings: ProxySettings | undefined): AxiosProxyConfig | undefined {
-  if (!proxySettings) {
-    return undefined;
+function isReadableStream(body: any): body is Readable {
+  return typeof body.pipe === "function";
+}
+
+declare type ProxyAgent = { isHttps: boolean; agent: http.Agent | https.Agent };
+export function createProxyAgent(requestUrl: string, proxySettings: ProxySettings, headers?: HttpHeaders): ProxyAgent {
+  const tunnelOptions: tunnel.HttpsOverHttpsOptions = {
+    proxy: {
+      host: proxySettings.host,
+      port: proxySettings.port,
+      headers: (headers && headers.rawHeaders()) || {}
+    }
+  };
+
+  if ((proxySettings.username && proxySettings.password)) {
+    tunnelOptions.proxy!.proxyAuth = `${proxySettings.username}:${proxySettings.password}`;
   }
 
-  const axiosAuthConfig = (proxySettings.username && proxySettings.password) ? {
-    username: proxySettings.username,
-    password: proxySettings.password
-  } : undefined;
+  const requestScheme = URLBuilder.parse(requestUrl).getScheme() || "";
+  const isRequestHttps = requestScheme.toLowerCase() === "https";
+  const proxyScheme = URLBuilder.parse(proxySettings.host).getScheme() || "";
+  const isProxyHttps = proxyScheme.toLowerCase() === "https";
 
-  const axiosProxyConfig: AxiosProxyConfig = {
-    host: proxySettings.host,
-    port: proxySettings.port,
-    auth: axiosAuthConfig
+  const proxyAgent = {
+    isHttps: isRequestHttps,
+    agent: createTunnel(isRequestHttps, isProxyHttps, tunnelOptions)
   };
 
-  return axiosProxyConfig;
+  return proxyAgent;
 }
 
-function isReadableStream(body: any): body is Readable {
-  return typeof body.pipe === "function";
+export function createTunnel(isRequestHttps: boolean, isProxyHttps: boolean, tunnelOptions: tunnel.HttpsOverHttpsOptions): http.Agent | https.Agent {
+  if (isRequestHttps && isProxyHttps) {
+    return tunnel.httpsOverHttps(tunnelOptions);
+  } else if (isRequestHttps && !isProxyHttps) {
+    return tunnel.httpsOverHttp(tunnelOptions);
+  } else if (!isRequestHttps && isProxyHttps) {
+    return tunnel.httpOverHttps(tunnelOptions);
+  } else {
+    return tunnel.httpOverHttp(tunnelOptions);
+  }
 }

lib/policies/proxyPolicy.ts

@@ -5,11 +5,47 @@ import { BaseRequestPolicy, RequestPolicy, RequestPolicyFactory, RequestPolicyOp
 import { HttpOperationResponse } from "../httpOperationResponse";
 import { ProxySettings } from "../serviceClient";
 import { WebResource } from "../webResource";
+import { Constants } from "../util/constants";
+import { URLBuilder } from "../url";
 
-export function proxyPolicy(proxySettings: ProxySettings): RequestPolicyFactory {
+function loadEnvironmentProxyValue(): string | undefined {
+  if (!process) {
+    return undefined;
+  }
+
+  if (process.env[Constants.HTTPS_PROXY]) {
+    return process.env[Constants.HTTPS_PROXY];
+  } else if (process.env[Constants.HTTPS_PROXY.toLowerCase()]) {
+    return process.env[Constants.HTTPS_PROXY.toLowerCase()];
+  } else if (process.env[Constants.HTTP_PROXY]) {
+    return process.env[Constants.HTTP_PROXY];
+  } else if (process.env[Constants.HTTP_PROXY.toLowerCase()]) {
+    return process.env[Constants.HTTP_PROXY.toLowerCase()];
+  }
+
+  return undefined;
+}
+
+export function getDefaultProxySettings(proxyUrl?: string): ProxySettings | undefined {
+  if (!proxyUrl) {
+    proxyUrl = loadEnvironmentProxyValue();
+    if (!proxyUrl) {
+      return undefined;
+    }
+  }
+
+  const parsedUrl = URLBuilder.parse(proxyUrl);
+  return {
+    host: parsedUrl.getScheme() + "://" + parsedUrl.getHost(),
+    port: Number.parseInt(parsedUrl.getPort() || "80")
+  };
+}
+
+
+export function proxyPolicy(proxySettings?: ProxySettings): RequestPolicyFactory {
   return {
     create: (nextPolicy: RequestPolicy, options: RequestPolicyOptions) => {
-      return new ProxyPolicy(nextPolicy, options, proxySettings);
+      return new ProxyPolicy(nextPolicy, options, proxySettings!);
     }
   };
 }

lib/serviceClient.ts

@@ -26,8 +26,8 @@ import { stringifyXML } from "./util/xml";
 import { RequestOptionsBase, RequestPrepareOptions, WebResource } from "./webResource";
 import { OperationResponse } from "./operationResponse";
 import { ServiceCallback } from "./util/utils";
+import { proxyPolicy, getDefaultProxySettings } from "./policies/proxyPolicy";
 import { throttlingRetryPolicy } from "./policies/throttlingRetryPolicy";
-import { proxyPolicy } from "./policies/proxyPolicy";
 
 
 /**
@@ -410,8 +410,9 @@ function createDefaultRequestPolicyFactories(credentials: ServiceClientCredentia
 
   factories.push(deserializationPolicy(options.deserializationContentTypes));
 
-  if (options.proxySettings) {
-    factories.push(proxyPolicy(options.proxySettings));
+  const proxySettings = options.proxySettings || getDefaultProxySettings();
+  if (proxySettings) {
+    factories.push(proxyPolicy(proxySettings));
   }
 
   return factories;

package.json

@@ -52,6 +52,7 @@
     "form-data": "^2.3.2",
     "tough-cookie": "^2.4.3",
     "tslib": "^1.9.2",
+    "tunnel": "0.0.6",
     "uuid": "^3.2.1",
     "xml2js": "^0.4.19"
   },
@@ -67,6 +68,7 @@
     "@types/semver": "^5.5.0",
     "@types/sinon": "^5.0.6",
     "@types/tough-cookie": "^2.3.3",
+    "@types/tunnel": "0.0.0",
     "@types/uuid": "^3.4.4",
     "@types/webpack": "^4.4.13",
     "@types/webpack-dev-middleware": "^2.0.2",

rollup.config.ts

@@ -24,13 +24,14 @@
    input: "./es/lib/msRest.js",
    external: [
      "axios",
-     "xml2js",
-     "tough-cookie",
-     "uuid/v4",
-     "tslib",
      "form-data",
+     "os",
      "stream",
-     "os"
+     "tough-cookie",
+     "tslib",
+     "tunnel",
+     "uuid/v4",
+     "xml2js",
    ],
    output: {
      file: "./dist/msRest.node.js",

test/axiosHttpClientTests.node.ts

@@ -0,0 +1,111 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+import "chai/register-should";
+import { should } from "chai";
+import tunnel from "tunnel";
+import https from "https";
+
+import { HttpHeaders } from "../lib/msRest";
+import { createTunnel, createProxyAgent } from "../lib/axiosHttpClient";
+
+describe("AxiosHttpClient", () => {
+    describe("createProxyAgent", () => {
+        type HttpsAgent = https.Agent & {
+            defaultPort: number | undefined,
+            options: {
+                proxy: tunnel.ProxyOptions
+            },
+            proxyOptions: tunnel.ProxyOptions
+        };
+
+        [
+            { proxy: "http", request: "ftp", port: undefined, isProxyHttps: false },
+            { proxy: "http", request: "http", port: undefined, isProxyHttps: false },
+            { proxy: "hTtp", request: "https", port: 443, isProxyHttps: true },
+            { proxy: "HTTPS", request: "http", port: undefined, isProxyHttps: false },
+            { proxy: "https", request: "hTTps", port: 443, isProxyHttps: true }
+        ].forEach(testCase => {
+            it(`should return ${testCase.isProxyHttps ? "HTTPS" : "HTTP"} proxy for ${testCase.proxy.toUpperCase()} proxy server and ${testCase.request.toUpperCase()} request`, function (done) {
+                const proxySettings = {
+                    host: `${testCase.proxy}://proxy.microsoft.com`,
+                    port: 8080
+                };
+                const requestUrl = `${testCase.request}://example.com`;
+
+                const proxyAgent = createProxyAgent(requestUrl, proxySettings);
+
+                proxyAgent.isHttps.should.equal(testCase.isProxyHttps);
+                const agent = proxyAgent.agent as HttpsAgent;
+                should().equal(agent.defaultPort, testCase.port);
+                agent.options.proxy.host!.should.equal(proxySettings.host);
+                agent.options.proxy.port!.should.equal(proxySettings.port);
+                done();
+            });
+        });
+
+        it("should copy headers correctly", function (done) {
+            const proxySettings = {
+                host: "http://proxy.microsoft.com",
+                port: 8080
+            };
+            const headers = new HttpHeaders({
+                "User-Agent": "Node.js"
+            });
+
+            const proxyAgent = createProxyAgent("http://example.com", proxySettings, headers);
+
+            const agent = proxyAgent.agent as HttpsAgent;
+            agent.proxyOptions.headers.should.contain({ "user-agent": "Node.js" });
+            done();
+        });
+    });
+
+    describe("createTunnel", () => {
+        const defaultProxySettings = {
+            host: "http://proxy.microsoft.com",
+            port: 8080
+        };
+
+        type HttpsAgent = https.Agent & {
+            defaultPort: number | undefined,
+            options: {
+                proxy: tunnel.ProxyOptions
+            }
+        };
+
+        [true, false].forEach(value => {
+            it(`returns HTTP agent for HTTP request and HTTP${value ? "S" : ""} proxy`, function () {
+                const tunnelConfig: tunnel.HttpsOverHttpsOptions = {
+                    proxy: {
+                        host: defaultProxySettings.host,
+                        port: defaultProxySettings.port,
+                        headers: {}
+                    }
+                };
+
+                const tunnel = createTunnel(false, value, tunnelConfig) as HttpsAgent;
+                tunnel.options.proxy.host!.should.equal(defaultProxySettings.host);
+                tunnel.options.proxy.port!.should.equal(defaultProxySettings.port);
+                should().not.exist(tunnel.defaultPort);
+            });
+        });
+
+        [true, false].forEach(value => {
+            it(`returns HTTPS agent for HTTPS request and HTTP${value ? "S" : ""} proxy`, function () {
+                const tunnelConfig: tunnel.HttpsOverHttpsOptions = {
+                    proxy: {
+                        host: defaultProxySettings.host,
+                        port: defaultProxySettings.port,
+                        headers: {}
+                    }
+                };
+
+                const tunnel = createTunnel(true, value, tunnelConfig) as HttpsAgent;
+                tunnel.options.proxy.host!.should.equal(defaultProxySettings.host);
+                tunnel.options.proxy.port!.should.equal(defaultProxySettings.port);
+                tunnel.defaultPort!.should.equal(443);
+            });
+        });
+    });
+});