Refactor proxy into own package, implement middleware pattern (#5136)

* renames

* Refactor proxy into own package, implement middleware pattern

don't need these mocha opts anymore

fix test

no more zunder

READMEs

fix test

* pass request by reference

* fix cors path

* Move replace_stream to proxy, concat-stream util in network

* Pin dependency versions

* Revert addDefaultPort behavior

* Add READMEs for proxy, network

* Update README.md

* eslint --fix

* set to null not undefined

* use delete and bump node types

* import cors from package now

* parse-domain@2.3.4

* proxy package needs common-tags

* move pumpify dep

* load through where it's needed, remove unused passthru_stream

* remove unneeded getbuffer call


Co-authored-by: Gleb Bahmutov <gleb.bahmutov@gmail.com>

circle.yml

@@ -297,7 +297,7 @@ jobs:
  - run: npm run all test -- --package https-proxy
  - run: npm run all test -- --package launcher
  - run: npm run all test -- --package network
- # how to pass Mocha reporter through zunder?
+ - run: npm run all test -- --package proxy
  - run: npm run all test -- --package reporter
  - run: npm run all test -- --package runner
  - run: npm run all test -- --package socket

package.json

@@ -86,7 +86,7 @@
  "@types/markdown-it": "0.0.9",
  "@types/mini-css-extract-plugin": "0.8.0",
  "@types/mocha": "5.2.7",
- "@types/node": "11.12.0",
+ "@types/node": "12.12.14",
  "@types/ramda": "0.25.47",
  "@types/react-dom": "16.9.4",
  "@types/request-promise": "4.1.42",
@@ -163,6 +163,7 @@
  "stop-only": "3.0.1",
  "strip-ansi": "4.0.0",
  "terminal-banner": "1.1.0",
+ "through": "2.3.8",
  "ts-node": "8.3.0",
  "typescript": "3.5.3",
  "vinyl-paths": "2.1.0"

packages/network/README.md

@@ -0,0 +1,44 @@
+# network
+
+This package contains networking-related classes and utilities.
+
+## Exports
+
+You can see a list of the modules exported from this package in [./lib/index.ts](./lib/index.ts). Here is a brief description of what's available:
+
+* `agent` is a HTTP/HTTPS [agent][1] with support for HTTP/HTTPS proxies and keepalive whenever possible
+* `allowDestroy` can be used to wrap a `net.Server` to add a `.destroy()` method
+* `blacklist` is a utility for matching glob blacklists
+* `concatStream` is a wrapper around [`concat-stream@1.6.2`][2] that makes it always yield a `Buffer`
+* `connect` contains utilities for making network connections, including `createRetryingSocket`
+* `cors` contains utilities for Cross-Origin Resource Sharing
+* `uri` contains utilities for URL parsing and formatting
+
+See the individual class files in [`./lib`](./lib) for more information.
+
+## Installing Dependencies
+
+```shell
+npm i
+```
+
+## Building
+
+Note: you should not ever need to build the .js files manually. `@packages/ts` provides require-time transpilation when in development.
+
+```shell
+npm run build-js
+```
+
+## Testing
+
+Tests are located in [`./test`](./test)
+
+To run tests:
+
+```shell
+npm run test
+```
+
+[1]: https://devdocs.io/node/http#http_class_http_agent
+[2]: https://github.com/maxogden/concat-stream/tree/v1.6.2

packages/network/lib/blacklist.ts

@@ -0,0 +1,18 @@
+import _ from 'lodash'
+import minimatch from 'minimatch'
+import { stripProtocolAndDefaultPorts } from './uri'
+
+export function matches (urlToCheck, blacklistHosts) {
+ // normalize into flat array
+ blacklistHosts = [].concat(blacklistHosts)
+
+ urlToCheck = stripProtocolAndDefaultPorts(urlToCheck)
+
+ // use minimatch against the url
+ // to see if any match
+ const matchUrl = (hostMatcher) => {
+ return minimatch(urlToCheck, hostMatcher)
+ }
+
+ return _.find(blacklistHosts, matchUrl)
+}

packages/network/lib/concat-stream.ts

@@ -0,0 +1,29 @@
+import _ from 'lodash'
+import _concatStream from 'concat-stream'
+
+type Callback = (buf: Buffer) => void
+type ConcatOpts = {
+ encoding?: string
+}
+
+/**
+ * Wrapper for `concat-stream` to handle empty streams.
+ */
+export const concatStream: typeof _concatStream = function (opts: Callback | ConcatOpts, cb?: Callback) {
+ let _cb: Callback = cb!
+
+ if (!_cb) {
+ _cb = opts as Callback
+ opts = {}
+ }
+
+ return _concatStream(opts as ConcatOpts, function (buf: Buffer) {
+ if (!_.get(buf, 'length')) {
+ // concat-stream can give an empty array if the stream has
+ // no data - just call the callback with an empty buffer
+ return _cb(Buffer.from(''))
+ }
+
+ return _cb(buf)
+ })
+}

packages/server/lib/util/cors.js → packages/network/lib/cors.ts

@@ -1,32 +1,43 @@
-const _ = require('lodash')
-const url = require('url')
-const uri = require('./uri')
-const debug = require('debug')('cypress:server:cors')
-const parseDomain = require('parse-domain')
+import _ from 'lodash'
+import * as uri from './uri'
+import debugModule from 'debug'
+import _parseDomain, { ParsedDomain } from 'parse-domain'
+
+const debug = debugModule('cypress:network:cors')
 
 const ipAddressRe = /^[\d\.]+$/
 
-function getSuperDomain (url) {
+type ParsedHost = {
+ port?: string
+ tld?: string
+ domain?: string
+}
+
+export function getSuperDomain (url) {
  const parsed = parseUrlIntoDomainTldPort(url)
 
  return _.compact([parsed.domain, parsed.tld]).join('.')
 }
 
-function _parseDomain (domain, options = {}) {
- return parseDomain(domain, _.defaults(options, {
+export function parseDomain (domain: string, options = {}) {
+ return _parseDomain(domain, _.defaults(options, {
  privateTlds: true,
  customTlds: ipAddressRe,
  }))
 }
 
-function parseUrlIntoDomainTldPort (str) {
- let { hostname, port, protocol } = url.parse(str)
+export function parseUrlIntoDomainTldPort (str) {
+ let { hostname, port, protocol } = uri.parse(str)
 
- if (port == null) {
+ if (!hostname) {
+ hostname = ''
+ }
+
+ if (!port) {
  port = protocol === 'https:' ? '443' : '80'
  }
 
- let parsed = _parseDomain(hostname)
+ let parsed: Partial<ParsedDomain> | null = parseDomain(hostname)
 
  // if we couldn't get a parsed domain
  if (!parsed) {
@@ -43,46 +54,33 @@ function parseUrlIntoDomainTldPort (str) {
  }
  }
 
- const obj = {}
+ const obj: ParsedHost = {}
 
  obj.port = port
  obj.tld = parsed.tld
  obj.domain = parsed.domain
- // obj.protocol = protocol
 
  debug('Parsed URL %o', obj)
 
  return obj
 }
 
-function urlMatchesOriginPolicyProps (urlStr, props) {
+export function urlMatchesOriginPolicyProps (urlStr, props) {
  // take a shortcut here in the case
  // where remoteHostAndPort is null
  if (!props) {
  return false
  }
 
- const parsedUrl = this.parseUrlIntoDomainTldPort(urlStr)
+ const parsedUrl = parseUrlIntoDomainTldPort(urlStr)
 
  // does the parsedUrl match the parsedHost?
  return _.isEqual(parsedUrl, props)
 }
 
-function urlMatchesOriginProtectionSpace (urlStr, origin) {
+export function urlMatchesOriginProtectionSpace (urlStr, origin) {
  const normalizedUrl = uri.addDefaultPort(urlStr).format()
  const normalizedOrigin = uri.addDefaultPort(origin).format()
 
  return _.startsWith(normalizedUrl, normalizedOrigin)
 }
-
-module.exports = {
- parseUrlIntoDomainTldPort,
-
- parseDomain: _parseDomain,
-
- getSuperDomain,
-
- urlMatchesOriginPolicyProps,
-
- urlMatchesOriginProtectionSpace,
-}

packages/network/lib/index.ts

@@ -1,9 +1,17 @@
 import agent from './agent'
+import * as blacklist from './blacklist'
 import * as connect from './connect'
-import { allowDestroy } from './allow-destroy'
+import * as cors from './cors'
+import * as uri from './uri'
 
 export {
  agent,
- allowDestroy,
+ blacklist,
  connect,
+ cors,
+ uri,
 }
+
+export { allowDestroy } from './allow-destroy'
+
+export { concatStream } from './concat-stream'

packages/server/lib/util/uri.js → packages/network/lib/uri.ts

@@ -5,8 +5,8 @@
 // node's url formatting algorithm (which acts pretty unexpectedly)
 // - https://nodejs.org/api/url.html#url_url_format_urlobject
 
-const _ = require('lodash')
-const url = require('url')
+import _ from 'lodash'
+import url from 'url'
 
 // yup, protocol contains a: ':' colon
 // at the end of it (-______________-)
@@ -25,9 +25,9 @@ const parseClone = (urlObject) => {
  return url.parse(_.clone(urlObject))
 }
 
-const parse = url.parse
+export const parse = url.parse
 
-const stripProtocolAndDefaultPorts = function (urlToCheck) {
+export function stripProtocolAndDefaultPorts (urlToCheck) {
  // grab host which is 'hostname:port' only
  const { host, hostname, port } = url.parse(urlToCheck)
 
@@ -41,20 +41,18 @@ const stripProtocolAndDefaultPorts = function (urlToCheck) {
  return host
 }
 
-const removePort = (urlObject) => {
+export function removePort (urlObject) {
  const parsed = parseClone(urlObject)
 
- // set host to null else
- // url.format(...) will ignore
- // the port property
+ // set host to undefined else url.format(...) will ignore the port property
  // https://nodejs.org/api/url.html#url_url_format_urlobject
- parsed.host = null
- parsed.port = null
+ delete parsed.host
+ delete parsed.port
 
  return parsed
 }
 
-const removeDefaultPort = function (urlToCheck) {
+export function removeDefaultPort (urlToCheck) {
  let parsed = parseClone(urlToCheck)
 
  if (portIsDefault(parsed.port)) {
@@ -64,33 +62,23 @@ const removeDefaultPort = function (urlToCheck) {
  return parsed
 }
 
-const addDefaultPort = function (urlToCheck) {
+export function addDefaultPort (urlToCheck) {
  const parsed = parseClone(urlToCheck)
 
  if (!parsed.port) {
  // unset host...
  // see above for reasoning
- parsed.host = null
- parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol]
+ delete parsed.host
+ if (parsed.protocol) {
+ parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol]
+ } else {
+ delete parsed.port
+ }
  }
 
  return parsed
 }
 
-const getPath = (urlToCheck) => {
+export function getPath (urlToCheck) {
  return url.parse(urlToCheck).path
 }
-
-module.exports = {
- parse,
-
- getPath,
-
- removePort,
-
- addDefaultPort,
-
- removeDefaultPort,
-
- stripProtocolAndDefaultPorts,
-}

packages/network/package.json

@@ -10,12 +10,15 @@
  },
  "dependencies": {
  "bluebird": "3.5.3",
+ "concat-stream": "1.6.2",
  "debug": "4.1.1",
  "lodash": "4.17.15",
+ "parse-domain": "2.3.4",
  "proxy-from-env": "1.0.0"
  },
  "devDependencies": {
  "@cypress/debugging-proxy": "2.0.1",
+ "@types/concat-stream": "1.6.0",
  "bin-up": "1.2.2",
  "express": "4.16.4",
  "request": "2.88.0",

packages/network/test/mocha.opts

@@ -1,5 +1,5 @@
 test/unit
 test/integration
---compilers ts:@packages/ts/register
+--compilers ts:@packages/ts/register,coffee:@packages/coffee/register
 --timeout 10000
 --recursive

packages/server/test/unit/blacklist_spec.js → packages/network/test/unit/blacklist_spec.ts

@@ -1,6 +1,5 @@
-require('../spec_helper')
-
-const blacklist = require(`${root}lib/util/blacklist`)
+import { blacklist } from '../..'
+import { expect } from 'chai'
 
 const hosts = [
  '*.google.com',
@@ -26,7 +25,7 @@ const matchesHost = (url, host) => {
  expect(blacklist.matches(url, hosts)).to.eq(host)
 }
 
-describe('lib/util/blacklist', () => {
+describe('lib/blacklist', () => {
  it('handles hosts, ports, wildcards', () => {
  matchesArray('https://mail.google.com/foo', true)
  matchesArray('https://shop.apple.com/bar', true)

packages/server/test/unit/cors_spec.js → packages/network/test/unit/cors_spec.ts

@@ -1,8 +1,7 @@
-require('../spec_helper')
+import { cors } from '../../lib'
+import { expect } from 'chai'
 
-const cors = require(`${root}lib/util/cors`)
-
-describe('lib/util/cors', () => {
+describe('lib/cors', () => {
  context('.parseUrlIntoDomainTldPort', () => {
  beforeEach(function () {
  this.isEq = (url, obj) => {