Reverse proxy with https support

.eslintrc.js

@@ -16,7 +16,9 @@ module.exports = {
     "indent": ["error", 2],
     "linebreak-style": [ "error", "unix"],
     "semi": ["error", "always"],
-    "eol-last": ["error", "always"]
+    "eol-last": ["error", "always"],
+    "keyword-spacing": [1],
+    "no-trailing-spaces": ["error", { "skipBlankLines": true }]
   },
   "overrides": [{
     "files": ["src/requestsDebugger.js", "src/commandLine.js", "test/**/*.test.js"],

.github/workflows/unit-tests.yml

@@ -0,0 +1,32 @@
+name: 'unit-tests'
+on:
+  pull_request:
+    paths:
+    - './**'
+    - '.github/workflows/unit-tests*'
+  push:
+    paths:
+    - './**'
+    - '.github/workflows/unit-tests*'
+
+jobs:
+  unit-tests:
+    runs-on: ${{ matrix.operating-system }}
+    strategy:
+      matrix:
+        operating-system: [ubuntu-latest, macos-latest]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set Node.js 4.9.1
+      uses: actions/setup-node@master
+      with:
+        node-version: 4.9.1
+
+    - name: npm install
+      working-directory: ./
+      run: npm install
+
+    - name: Lint and Unit tests
+      working-directory: ./
+      run: npm run test

.gitignore

@@ -108,3 +108,9 @@ dist
 RequestsDebugger-Mac
 RequestsDebugger.exe
 RequestsDebugger-Linux*
+
+# VS code settings
+.vscode/
+
+# Others
+.DS_Store

README.md

@@ -1,8 +1,17 @@
 ## Requests Debugger
+
+<p align="center">
+  <a href="https://browserstack.com"><img alt="BrowserStack Logo" src="https://d98b8t1nnulk5.cloudfront.net/production/images/layout/logo-invoice.svg"></a>
+</p>
+
+<p align="center">
+  <a href="https://github.com/browserstack/requests-debugger/actions?query=workflow%3Aunit-tests"><img alt="unit-tests build status" src="https://github.com/browserstack/requests-debugger/workflows/unit-tests/badge.svg"></a>
+</p>
+
 ### Tool for debugging client side failure of requests, leading to requests getting dropped or not reaching BrowserStack.
 
 ## Features
-- Proxy Server to intercept requests fired by client bindings to keep track of their flow.
+- Server to intercept requests fired by client bindings to keep track of their flow.
 - Connectivity Checker : To check for the reachability of BrowserStack components, i.e. Rails & Hub.
 - Multi-Platform Stats Compatibility : Ability to collect stats of CPU, Network & Memory Stats.
 - Retry Mechanism in case a request fails at the client side
@@ -13,6 +22,8 @@
   - Start Requests Debugger with the required arguments: `npm run start -- <args>`. 
   - Supported `args`:
     - `--port <port>`: Port on which the Requests Debugger Tool's Proxy will run. Default: 9687
+    - `--reverse-proxy-port <port>`: Port on which the Requests Debugger Tool's Reverse Proxy will run. Default: 9688
+    - `--scheme <https/http>`: Scheme for requests to browserstack. Default: https
     - `--proxy-host <hostname>`: Hostname of the Upstream Proxy
     - `--proxy-port <port>`: Port of the Upstream Proxy. Default: 3128 (if hostname is provided)
     - `--proxy-user <username>`: Username for auth of the Upstream Proxy
@@ -30,7 +41,8 @@
     - Windows: `RequestsDebugger.exe <args>`
 
 ## How to use
-- Since the tool acts like a proxy, you will have to set the proxy to be used by your client binding to `localhost:9687`. i.e.
+- To use tool as reverse proxy, you will have to replace `hub-cloud.browserstack.com` in hub url with `localhost:9688`.
+- To use tool as a proxy, you will have to set the proxy to be used by your client binding to `localhost:9687`. i.e.
   - For Java:
     - ```
       System.getProperties().put("http.proxyHost", "localhost");
@@ -40,7 +52,7 @@
     - Set your system's env variable `http_proxy=localhost:9687` and Ruby's Selenium Client Binding will pick the value. Or,
     - Run you test by giving the environment variable to your command itself, i.e. `http_proxy=localhost:9687 ruby <your_script.rb>`
   - Similarly, you can also set proxy for other client bindings.
-  
+
 ## Steps to build the executables
 - Linux
   - `npm run build:linux`

config/constants.js

@@ -1,6 +1,12 @@
-module.exports.VERSION = '1.0.0';
-module.exports.HUB_STATUS_URL = 'http://hub-cloud.browserstack.com/wd/hub/status';
-module.exports.RAILS_AUTOMATE = 'http://automate.browserstack.com';
+module.exports.VERSION = '1.1.0';
+module.exports.BS_DOMAIN = 'browserstack.com';
+module.exports.HUB_HOST = 'hub-cloud.' + this.BS_DOMAIN;
+module.exports.RAILS_HOST = 'automate.' + this.BS_DOMAIN;
+module.exports.HUB_STATUS_PATH = '/wd/hub/status';
+module.exports.HUB_STATUS_URL = 'http://' + this.HUB_HOST + this.HUB_STATUS_PATH;
+module.exports.HUB_STATUS_URL_HTTPS = 'https://' + this.HUB_HOST + this.HUB_STATUS_PATH;
+module.exports.RAILS_AUTOMATE = 'http://' + this.RAILS_HOST;
+module.exports.RAILS_AUTOMATE_HTTPS = 'https://' + this.RAILS_HOST;
 module.exports.CONNECTIVITY_REQ_TIMEOUT = 30000;
 module.exports.DEFAULT_PROXY_PORT = 3128;
 module.exports.CUSTOM_ERROR_RESPONSE_CODE = 502;
@@ -10,6 +16,8 @@ module.exports.PORTS = {
   MAX: 65535,
   MIN: 1
 };
+
+module.exports.LINE_LENGTH = 70;
 module.exports.PROTOCOL_REGEX = /(^\w+:|^)\/\//;
 
 module.exports.LOGS = Object.freeze({
@@ -23,13 +31,15 @@ module.exports.LOGS = Object.freeze({
 
 module.exports.RdGlobalConfig = {
   RETRY_DELAY: 1000, // in ms
-  RD_HANDLER_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687,
+  RD_HANDLER_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687,
+  RD_HANDLER_REVERSE_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8788 : 9688,
   CLIENT_REQ_TIMEOUT: 260000, // in ms
+  SCHEME: 'https'
 };
 
 module.exports.COMMON = Object.freeze({
-  PING_HUB: 'ping -c 5 hub-cloud.browserstack.com',
-  PING_AUTOMATE: 'ping -c 5 automate.browserstack.com'
+  PING_HUB: 'ping -c 5 ' + this.HUB_HOST,
+  PING_AUTOMATE: 'ping -c 5 ' + this.RAILS_HOST
 });
 
 module.exports.MAC = Object.freeze({
@@ -49,8 +59,8 @@ module.exports.WIN = Object.freeze({
   NETSTAT_ROUTING_TABLE: 'netstat -r',
   IPCONFIG_ALL: 'ipconfig /all',
   SWAP_USAGE: 'pagefile get AllocatedBaseSize, CurrentUsage', // this is a WMIC command. Prefix with WMIC Path
-  PING_HUB: 'ping -n 5 hub-cloud.browserstack.com',
-  PING_AUTOMATE: 'ping -n 5 automate.browserstack.com',
+  PING_HUB: 'ping -n 5 ' + this.HUB_HOST,
+  PING_AUTOMATE: 'ping -n 5 ' + this.RAILS_HOST,
   LOAD_PERCENTAGE: 'cpu get loadpercentage', // prefix wmic path
 });
 
@@ -78,8 +88,10 @@ module.exports.STATIC_MESSAGES = Object.freeze({
   CHECK_NETWORK_STATS: 'Stats : Checking Network Stats',
   CHECK_MEMORY_STATS: 'Stats : Checking Memory Stats',
   CHECK_CONNECTIVITY: 'Checks : Checking Connectivity With BrowserStack',
-  ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool Proxy: ',
-  TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Started on Port: ',
+  ERR_STARTING_TOOL_PROXY: 'Error in starting Requests Debugger Tool Proxy: ',
+  ERR_STARTING_TOOL_REVERSE_PROXY: 'Error in starting Requests Debugger Tool Reverse Proxy: ',
+  TOOL_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Server Started on Port: ',
+  TOOL_REVESE_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Server Started on Port: ',
   CPU_STATS_COLLECTED: 'Stats : Initial CPU Stats Collected',
   NETWORK_STATS_COLLECTED: 'Stats : Initial Network Stats Collected',
   MEMORY_STATS_COLLECTED: 'Stats : Initial Memory Stats Collected',

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "NODE_ENV=prod node src/requestsDebugger.js",
     "lint": "./node_modules/.bin/eslint 'src/*' 'test/*' 'config/*.js'",
-    "test": "npm run lint; NODE_ENV=test nyc --reporter=html ./node_modules/mocha/bin/mocha 'test/**/*.test.js'",
+    "test": "npm run lint; NODE_ENV=test ./node_modules/nyc/bin/nyc.js --reporter=html ./node_modules/mocha/bin/mocha 'test/**/*.test.js'",
     "build:mac": "npm install; ./node_modules/pkg/lib-es5/bin.js -t node4-macos-x64 src/requestsDebugger.js; mv requestsDebugger RequestsDebugger-Mac",
     "build:linux-x86": "npm install; ./node_modules/pkg/lib-es5/bin.js -t node4-linux-x86 src/requestsDebugger.js; mv requestsDebugger RequestsDebugger-Linux-x86",
     "build:linux-x64": "npm install; ./node_modules/pkg/lib-es5/bin.js -t node4-linux-x64 src/requestsDebugger.js; mv requestsDebugger RequestsDebugger-Linux-x64",
@@ -34,6 +34,8 @@
     "sinon": "7.5.0"
   },
   "dependencies": {
+    "http-proxy-agent": "^2.1.0",
+    "https-proxy-agent": "^2.1.0",
     "uuid": "3.4.0",
     "winston": "2.4.4"
   },

src/commandLine.js

@@ -2,8 +2,8 @@
  * Command Line Manager to parse the command line arguments
  * and set the necessary fields in RdGlobalConfig.
  */
-
 var constants = require('../config/constants');
+
 var RdGlobalConfig = constants.RdGlobalConfig;
 
 var CommandLineManager = {
@@ -13,21 +13,25 @@ var CommandLineManager = {
                      + "\n"
                      + "Usage: RequestsDebugger [ARGUMENTS]\n\n"
                      + "ARGUMENTS:\n"
-                     + "  --port            <port>                    : Port on which the Requests Debugger Tool's Proxy will run\n"
-                     + "                                                Default: " + RdGlobalConfig.RD_HANDLER_PORT + "\n"
-                     + "  --proxy-host      <hostname>                : Hostname of the Upstream Proxy\n"
-                     + "  --proxy-port      <port>                    : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n"
-                     + "  --proxy-user      <username>                : Username for auth of the Upstream Proxy\n"
-                     + "  --proxy-pass      <password>                : Password for auth of the Upstream Proxy\n"
-                     + "  --retry-delay     <milliseconds>            : Delay for the retry of a failed request. Default: " + RdGlobalConfig.RETRY_DELAY + "ms\n"
-                     + "  --request-timeout <milliseconds>            : Hard timeout for the requests being fired by the tool before receiving any response\n"
-                     + "                                                Default: " + RdGlobalConfig.CLIENT_REQ_TIMEOUT + "ms\n"
-                     + "  --logs-path       <relative/absolute path>  : Directory where the '" + constants.LOGS_FOLDER + "' folder will be created\n"
-                     + "                                                for storing logs. Default: Current Working Directory\n"
-                     + "  --del-logs                                  : Deletes any existing logs from the " + constants.LOGS_FOLDER + "/ directory and initializes\n"
-                     + "                                                new files for logging\n"
-                     + "  --help                                      : Help for Requests Debugger Tool\n"
-                     + "  --version                                   : Version of the Requests Debugger Tool\n";
+                     + "  --proxy-port         <port>                    : Port on which the Requests Debugger Tool's Proxy will run\n"
+                     + "                                                   Default: " + RdGlobalConfig.RD_HANDLER_PROXY_PORT + "\n"
+                     + "  --reverse-proxy-port <port>                    : Port on which the Requests Debugger Tool's Reverse Proxy will run\n"
+                     + "                                                   Default: " + RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT + "\n"
+                     + "  --scheme             <https/http>              : Scheme for requests to browserstack.\n"
+                     + "                                                   Default: " + RdGlobalConfig.SCHEME + "\n"
+                     + "  --proxy-host         <hostname>                : Hostname of the Upstream Proxy\n"
+                     + "  --proxy-port         <port>                    : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n"
+                     + "  --proxy-user         <username>                : Username for auth of the Upstream Proxy\n"
+                     + "  --proxy-pass         <password>                : Password for auth of the Upstream Proxy\n"
+                     + "  --retry-delay        <milliseconds>            : Delay for the retry of a failed request. Default: " + RdGlobalConfig.RETRY_DELAY + "ms\n"
+                     + "  --request-timeout    <milliseconds>            : Hard timeout for the requests being fired by the tool before receiving any response\n"
+                     + "                                                   Default: " + RdGlobalConfig.CLIENT_REQ_TIMEOUT + "ms\n"
+                     + "  --logs-path          <relative/absolute path>  : Directory where the '" + constants.LOGS_FOLDER + "' folder will be created\n"
+                     + "                                                   for storing logs. Default: Current Working Directory\n"
+                     + "  --del-logs                                     : Deletes any existing logs from the " + constants.LOGS_FOLDER + "/ directory and initializes\n"
+                     + "                                                   new files for logging\n"
+                     + "  --help                                         : Help for Requests Debugger Tool\n"
+                     + "  --version                                      : Version of the Requests Debugger Tool\n";
 
     console.log(helpOutput);
   },
@@ -38,7 +42,7 @@ var CommandLineManager = {
 
   /**
    * Processes the args from the given input array and sets the global config.
-   * @param {Array<String>} argv 
+   * @param {Array<String>} argv
    */
   processArgs: function (argv) {
 
@@ -67,7 +71,7 @@ var CommandLineManager = {
       if (CommandLineManager.validArgValue(argv[index + 1])) {
         var probablePort = parseInt(argv[index + 1]);
         if (!isNaN(probablePort) && (probablePort <= constants.PORTS.MAX) && (probablePort >= constants.PORTS.MIN)) {
-          RdGlobalConfig.RD_HANDLER_PORT = probablePort;
+          RdGlobalConfig.RD_HANDLER_PROXY_PORT = probablePort;
         } else {
           console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX);
           invalidArgs.add('--port');
@@ -79,6 +83,24 @@ var CommandLineManager = {
       }
     }
 
+    // port for Requests Debugger Reverse Proxy
+    index = argv.indexOf('--reverse-proxy-port');
+    if (index !== -1) {
+      if (CommandLineManager.validArgValue(argv[index + 1])) {
+        var probableReverseProxyPort = parseInt(argv[index + 1]);
+        if (!isNaN(probableReverseProxyPort) && (probableReverseProxyPort <= constants.PORTS.MAX) && (probableReverseProxyPort >= constants.PORTS.MIN)) {
+          RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT = probableReverseProxyPort;
+        } else {
+          console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX);
+          invalidArgs.add('--reverse-proxy-port');
+        }
+        argv.splice(index, 2);
+      } else {
+        invalidArgs.add('--reverse-proxy-port');
+        argv.splice(index, 1);
+      }
+    }
+
     // delay for retries in case of request failures
     index = argv.indexOf('--retry-delay');
     if (index !== -1) {
@@ -115,12 +137,34 @@ var CommandLineManager = {
       }
     }
 
+    // process proxy host
+    index = argv.indexOf('--scheme');
+    if (index !== -1) {
+      if (CommandLineManager.validArgValue(argv[index + 1])) {
+        var scheme = argv[index + 1];
+        if (!(scheme === 'http' || scheme === 'https')){
+          console.log("\nScheme can only be http/https");
+          invalidArgs.add('--scheme');
+          argv.splice(index, 2);
+        }
+        else {
+          RdGlobalConfig.SCHEME = scheme;
+          argv.splice(index, 2);
+        }
+      } else {
+        invalidArgs.add('--scheme');
+        argv.splice(index, 1);
+      }
+    }
+
     // process proxy host
     index = argv.indexOf('--proxy-host');
     if (index !== -1) {
       if (CommandLineManager.validArgValue(argv[index + 1])) {
         var host = argv[index + 1];
-        host = host.replace(constants.PROTOCOL_REGEX, '');
+        if (host.lastIndexOf("http") !== 0){
+          host = 'http://' + host;
+        }
         RdGlobalConfig.proxy = RdGlobalConfig.proxy || {};
         RdGlobalConfig.proxy.host = host;
         argv.splice(index, 2);
@@ -151,7 +195,7 @@ var CommandLineManager = {
         argv.splice(index, 1);
       }
     }
-    
+
     // if proxy port value in invalid or doesn't exist and host exists, set the default value
     if (RdGlobalConfig.proxy && RdGlobalConfig.proxy.host && (invalidArgs.has('--proxy-port') || !RdGlobalConfig.proxy.port)) {
       console.log('\nSetting Default Proxy Port:', constants.DEFAULT_PROXY_PORT, '\n');
@@ -190,7 +234,7 @@ var CommandLineManager = {
         argv.splice(index, 1);
       }
     }
-    
+
     // if proxy pass is invalid or doesn't exist and username exists, set the password as empty
     if (RdGlobalConfig.proxy && RdGlobalConfig.proxy.username && (invalidArgs.has('--proxy-pass') || !RdGlobalConfig.proxy.password)) {
       console.log('Setting Proxy Password as Empty\n');