Refactor proxy into own package, implement middleware pattern

circle.yml

@@ -262,6 +262,7 @@ jobs:
  - run: npm run all test -- --package https-proxy
  - run: npm run all test -- --package launcher
  - run: npm run all test -- --package network
+ - run: npm run all test -- --package proxy
  # how to pass Mocha reporter through zunder?
  - run: npm run all test -- --package reporter
  - run: npm run all test -- --package runner

packages/network/lib/blacklist.ts

@@ -1,12 +1,12 @@
-const _ = require('lodash')
-const minimatch = require('minimatch')
-const uri = require('./uri')
+import _ from 'lodash'
+import minimatch from 'minimatch'
+import { stripProtocolAndDefaultPorts } from './uri'
 
-const matches = function (urlToCheck, blacklistHosts) {
- //# normalize into flat array
+export function matches (urlToCheck, blacklistHosts) {
+ // normalize into flat array
  blacklistHosts = [].concat(blacklistHosts)
 
- urlToCheck = uri.stripProtocolAndDefaultPorts(urlToCheck)
+ urlToCheck = stripProtocolAndDefaultPorts(urlToCheck)
 
  // use minimatch against the url
  // to see if any match
@@ -16,7 +16,3 @@ const matches = function (urlToCheck, blacklistHosts) {
 
  return _.find(blacklistHosts, matchUrl)
 }
-
-module.exports = {
- matches,
-}

packages/network/lib/cors.ts

@@ -1,68 +1,76 @@
-const _ = require('lodash')
-const url = require('url')
-const uri = require('./uri')
-const debug = require('debug')('cypress:server:cors')
-const parseDomain = require('parse-domain')
+import _ from 'lodash'
+import * as uri from './uri'
+import debugModule from 'debug'
+import parseDomain, { ParsedDomain } from 'parse-domain'
+
+const debug = debugModule('cypress:network:cors')
 
 const ipAddressRe = /^[\d\.]+$/
 
-module.exports = {
- parseUrlIntoDomainTldPort (str) {
- let { hostname, port, protocol } = url.parse(str)
+type ParsedHost = {
+ port?: string
+ tld?: string
+ domain?: string
+}
 
- if (port == null) {
- port = protocol === 'https:' ? '443' : '80'
- }
+export function parseUrlIntoDomainTldPort (str) {
+ let { hostname, port, protocol } = uri.parse(str)
+
+ if (!hostname) {
+ hostname = ''
+ }
+
+ if (!port) {
+ port = protocol === 'https:' ? '443' : '80'
+ }
 
- let parsed = parseDomain(hostname, {
- privateTlds: true, // use the public suffix
- customTlds: ipAddressRe,
- })
-
- // if we couldn't get a parsed domain
- if (!parsed) {
- // then just fall back to a dumb check
- // based on assumptions that the tld
- // is the last segment after the final
- // '.' and that the domain is the segment
- // before that
- const segments = hostname.split('.')
-
- parsed = {
- tld: segments[segments.length - 1] || '',
- domain: segments[segments.length - 2] || '',
- }
+ let parsed : Partial<ParsedDomain> | null = parseDomain(hostname, {
+ privateTlds: true, // use the public suffix
+ customTlds: ipAddressRe,
+ })
+
+ // if we couldn't get a parsed domain
+ if (!parsed) {
+ // then just fall back to a dumb check
+ // based on assumptions that the tld
+ // is the last segment after the final
+ // '.' and that the domain is the segment
+ // before that
+ const segments = hostname.split('.')
+
+ parsed = {
+ tld: segments[segments.length - 1] || '',
+ domain: segments[segments.length - 2] || '',
  }
+ }
 
-  const obj = {}
+ const obj: ParsedHost = {}
 
- obj.port = port
- obj.tld = parsed.tld
- obj.domain = parsed.domain
- // obj.protocol = protocol
+ obj.port = port
+ obj.tld = parsed.tld
+ obj.domain = parsed.domain
 
-  debug('Parsed URL %o', obj)
+ debug('Parsed URL %o', obj)
 
-  return obj
- },
+ return obj
+}
 
-  urlMatchesOriginPolicyProps (urlStr, props) {
-  // take a shortcut here in the case
-  // where remoteHostAndPort is null
-  if (!props) {
-  return false
-  }
+export function urlMatchesOriginPolicyProps (urlStr, props) {
+ // take a shortcut here in the case
+ // where remoteHostAndPort is null
+ if (!props) {
+ return false
+ }
 
-  const parsedUrl = this.parseUrlIntoDomainTldPort(urlStr)
+ const parsedUrl = parseUrlIntoDomainTldPort(urlStr)
 
-  // does the parsedUrl match the parsedHost?
-  return _.isEqual(parsedUrl, props)
- },
+ // does the parsedUrl match the parsedHost?
+ return _.isEqual(parsedUrl, props)
+}
 
-  urlMatchesOriginProtectionSpace (urlStr, origin) {
-  const normalizedUrl = uri.addDefaultPort(urlStr).format()
-  const normalizedOrigin = uri.addDefaultPort(origin).format()
+export function urlMatchesOriginProtectionSpace (urlStr, origin) {
+ const normalizedUrl = uri.addDefaultPort(urlStr).format()
+ const normalizedOrigin = uri.addDefaultPort(origin).format()
 
- return _.startsWith(normalizedUrl, normalizedOrigin)
- },
+ return _.startsWith(normalizedUrl, normalizedOrigin)
 }

packages/network/lib/index.ts

@@ -1,9 +1,15 @@
 import agent from './agent'
-import * as connect from './connect'
 import { allowDestroy } from './allow-destroy'
+import * as blacklist from './blacklist'
+import * as connect from './connect'
+import * as cors from './cors'
+import * as uri from './uri'
 
 export {
  agent,
  allowDestroy,
+ blacklist,
  connect,
+ cors,
+ uri,
 }

packages/server/lib/util/uri.js → packages/network/lib/uri.ts

@@ -5,8 +5,8 @@
 // node's url formatting algorithm (which acts pretty unexpectedly)
 // - https://nodejs.org/api/url.html#url_url_format_urlobject
 
-const _ = require('lodash')
-const url = require('url')
+import _ from 'lodash'
+import url from 'url'
 
 // yup, protocol contains a: ':' colon
 // at the end of it (-______________-)
@@ -25,9 +25,9 @@ const parseClone = (urlObject) => {
  return url.parse(_.clone(urlObject))
 }
 
-const parse = url.parse
+export const parse = url.parse
 
-const stripProtocolAndDefaultPorts = function (urlToCheck) {
+export function stripProtocolAndDefaultPorts (urlToCheck) {
  // grab host which is 'hostname:port' only
  const { host, hostname, port } = url.parse(urlToCheck)
 
@@ -41,20 +41,18 @@ const stripProtocolAndDefaultPorts = function (urlToCheck) {
  return host
 }
 
-const removePort = (urlObject) => {
+export function removePort (urlObject) {
  const parsed = parseClone(urlObject)
 
- // set host to null else
- // url.format(...) will ignore
- // the port property
+ // set host to undefined else url.format(...) will ignore the port property
  // https://nodejs.org/api/url.html#url_url_format_urlobject
- parsed.host = null
- parsed.port = null
+ parsed.host = undefined
+ parsed.port = undefined
 
  return parsed
 }
 
-const removeDefaultPort = function (urlToCheck) {
+export function removeDefaultPort (urlToCheck) {
  let parsed = parseClone(urlToCheck)
 
  if (portIsDefault(parsed.port)) {
@@ -64,33 +62,19 @@ const removeDefaultPort = function (urlToCheck) {
  return parsed
 }
 
-const addDefaultPort = function (urlToCheck) {
+export function addDefaultPort (urlToCheck) {
  const parsed = parseClone(urlToCheck)
 
  if (!parsed.port) {
  // unset host...
  // see above for reasoning
- parsed.host = null
- parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol]
+ parsed.host = undefined
+ parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol || 'http:']
  }
 
  return parsed
 }
 
-const getPath = (urlToCheck) => {
+export function getPath (urlToCheck) {
  return url.parse(urlToCheck).path
 }
-
-module.exports = {
- parse,
-
- getPath,
-
- removePort,
-
- addDefaultPort,
-
- removeDefaultPort,
-
- stripProtocolAndDefaultPorts,
-}

packages/network/package.json

@@ -12,6 +12,7 @@
  "bluebird": "3.5.3",
  "debug": "4.1.1",
  "lodash": "4.17.15",
+ "parse-domain": "2.0.0",
  "proxy-from-env": "1.0.0"
  },
  "devDependencies": {

packages/network/test/mocha.opts

@@ -1,5 +1,5 @@
 test/unit
 test/integration
---compilers ts:@packages/ts/register
+--compilers coffee:@packages/coffee/register,ts:@packages/ts/register
 --timeout 10000
 --recursive

packages/server/test/unit/blacklist_spec.coffee → packages/network/test/unit/blacklist_spec.coffee

@@ -1,6 +1,5 @@
-require("../spec_helper")
-
-blacklist = require("#{root}lib/util/blacklist")
+{ blacklist } = require("../..")
+{ expect } = require("chai")
 
 hosts = [
  "*.google.com"
@@ -21,7 +20,7 @@ matchesArray = (url, val) ->
 matchesHost = (url, host) ->
  expect(blacklist.matches(url, hosts)).to.eq(host)
 
-describe "lib/util/blacklist", ->
+describe "lib/blacklist", ->
  it "handles hosts, ports, wildcards", ->
  matchesArray("https://mail.google.com/foo", true)
  matchesArray("https://shop.apple.com/bar", true)

packages/server/test/unit/cors_spec.coffee → packages/network/test/unit/cors_spec.coffee

@@ -1,8 +1,7 @@
-require("../spec_helper")
+{ cors } = require("../..")
+{ expect } = require("chai")
 
-cors = require("#{root}lib/util/cors")
-
-describe "lib/util/cors", ->
+describe "lib/cors", ->
  context ".parseUrlIntoDomainTldPort", ->
  beforeEach ->
  @isEq = (url, obj) ->

packages/proxy/index.js

@@ -0,0 +1,5 @@
+if (process.env.CYPRESS_ENV !== 'production') {
+ require('@packages/ts/register')
+}
+
+module.exports = require('./lib')

packages/proxy/lib/http/error-middleware.ts

@@ -0,0 +1,48 @@
+import _ from 'lodash'
+import debugModule from 'debug'
+import { HttpMiddleware } from '.'
+import { Readable } from 'stream'
+import { Request } from 'request'
+
+const debug = debugModule('cypress:proxy:http:error-middleware')
+
+type ErrorMiddleware = HttpMiddleware<{
+ error: Error
+ incomingResStream?: Readable
+ outgoingReq?: Request
+}>
+
+const LogError : ErrorMiddleware = function () {
+ debug('error proxying request %o', _.pick(this, 'error', 'req', 'res', 'incomingRes', 'outgoingReq', 'incomingResStream'))
+ this.next()
+}
+
+export const AbortRequest : ErrorMiddleware = function () {
+ if (this.outgoingReq) {
+ debug('aborting outgoingReq')
+ this.outgoingReq.abort()
+ }
+
+ this.next()
+}
+
+export const UnpipeResponse : ErrorMiddleware = function () {
+ if (this.incomingResStream) {
+ debug('unpiping resStream from response')
+ this.incomingResStream.unpipe()
+ }
+
+ this.next()
+}
+
+export const DestroyResponse : ErrorMiddleware = function () {
+ this.res.destroy()
+ this.end()
+}
+
+export default {
+ LogError,
+ AbortRequest,
+ UnpipeResponse,
+ DestroyResponse,
+}