index on refactor-proxy: 7b85344 Fix proxying HTTPS requests to IP ad…

…dresses (#4947)

circle.yml

@@ -262,6 +262,7 @@ jobs:
       - run: npm run all test -- --package https-proxy
       - run: npm run all test -- --package launcher
       - run: npm run all test -- --package network
+      - run: npm run all test -- --package proxy
       # how to pass Mocha reporter through zunder?
       - run: npm run all test -- --package reporter
       - run: npm run all test -- --package runner

packages/network/lib/blacklist.ts

@@ -0,0 +1,18 @@
+import _ from 'lodash'
+import minimatch from 'minimatch'
+import { stripProtocolAndDefaultPorts } from './uri'
+
+export function matches (urlToCheck, blacklistHosts) {
+  // normalize into flat array
+  blacklistHosts = [].concat(blacklistHosts)
+
+  urlToCheck = stripProtocolAndDefaultPorts(urlToCheck)
+
+  // use minimatch against the url
+  // to see if any match
+  const matchUrl = (hostMatcher) => {
+    return minimatch(urlToCheck, hostMatcher)
+  }
+
+  return _.find(blacklistHosts, matchUrl)
+}

packages/network/lib/cors.ts

@@ -0,0 +1,76 @@
+import _ from 'lodash'
+import * as uri from './uri'
+import debugModule from 'debug'
+import parseDomain, { ParsedDomain } from 'parse-domain'
+
+const debug = debugModule('cypress:network:cors')
+
+const ipAddressRe = /^[\d\.]+$/
+
+type ParsedHost = {
+  port?: string
+  tld?: string
+  domain?: string
+}
+
+export function parseUrlIntoDomainTldPort (str) {
+  let { hostname, port, protocol } = uri.parse(str)
+
+  if (!hostname) {
+    hostname = ''
+  }
+
+  if (!port) {
+    port = protocol === 'https:' ? '443' : '80'
+  }
+
+  let parsed : Partial<ParsedDomain> | null = parseDomain(hostname, {
+    privateTlds: true, // use the public suffix
+    customTlds: ipAddressRe,
+  })
+
+  // if we couldn't get a parsed domain
+  if (!parsed) {
+    // then just fall back to a dumb check
+    // based on assumptions that the tld
+    // is the last segment after the final
+    // '.' and that the domain is the segment
+    // before that
+    const segments = hostname.split('.')
+
+    parsed = {
+      tld: segments[segments.length - 1] || '',
+      domain: segments[segments.length - 2] || '',
+    }
+  }
+
+  const obj: ParsedHost = {}
+
+  obj.port = port
+  obj.tld = parsed.tld
+  obj.domain = parsed.domain
+
+  debug('Parsed URL %o', obj)
+
+  return obj
+}
+
+export function urlMatchesOriginPolicyProps (urlStr, props) {
+  // take a shortcut here in the case
+  // where remoteHostAndPort is null
+  if (!props) {
+    return false
+  }
+
+  const parsedUrl = parseUrlIntoDomainTldPort(urlStr)
+
+  // does the parsedUrl match the parsedHost?
+  return _.isEqual(parsedUrl, props)
+}
+
+export function urlMatchesOriginProtectionSpace (urlStr, origin) {
+  const normalizedUrl = uri.addDefaultPort(urlStr).format()
+  const normalizedOrigin = uri.addDefaultPort(origin).format()
+
+  return _.startsWith(normalizedUrl, normalizedOrigin)
+}

packages/network/lib/index.ts

@@ -1,9 +1,15 @@
 import agent from './agent'
-import * as connect from './connect'
 import { allowDestroy } from './allow-destroy'
+import * as blacklist from './blacklist'
+import * as connect from './connect'
+import * as cors from './cors'
+import * as uri from './uri'
 
 export {
   agent,
   allowDestroy,
+  blacklist,
   connect,
+  cors,
+  uri,
 }

packages/server/lib/util/uri.js → packages/network/lib/uri.ts

@@ -5,8 +5,8 @@
 // node's url formatting algorithm (which acts pretty unexpectedly)
 // - https://nodejs.org/api/url.html#url_url_format_urlobject
 
-const _ = require('lodash')
-const url = require('url')
+import _ from 'lodash'
+import url from 'url'
 
 // yup, protocol contains a: ':' colon
 // at the end of it (-______________-)
@@ -25,9 +25,9 @@ const parseClone = (urlObject) => {
   return url.parse(_.clone(urlObject))
 }
 
-const parse = url.parse
+export const parse = url.parse
 
-const stripProtocolAndDefaultPorts = function (urlToCheck) {
+export function stripProtocolAndDefaultPorts (urlToCheck) {
   // grab host which is 'hostname:port' only
   const { host, hostname, port } = url.parse(urlToCheck)
 
@@ -41,20 +41,18 @@ const stripProtocolAndDefaultPorts = function (urlToCheck) {
   return host
 }
 
-const removePort = (urlObject) => {
+export function removePort (urlObject) {
   const parsed = parseClone(urlObject)
 
-  // set host to null else
-  // url.format(...) will ignore
-  // the port property
+  // set host to undefined else url.format(...) will ignore the port property
   // https://nodejs.org/api/url.html#url_url_format_urlobject
-  parsed.host = null
-  parsed.port = null
+  parsed.host = undefined
+  parsed.port = undefined
 
   return parsed
 }
 
-const removeDefaultPort = function (urlToCheck) {
+export function removeDefaultPort (urlToCheck) {
   let parsed = parseClone(urlToCheck)
 
   if (portIsDefault(parsed.port)) {
@@ -64,33 +62,19 @@ const removeDefaultPort = function (urlToCheck) {
   return parsed
 }
 
-const addDefaultPort = function (urlToCheck) {
+export function addDefaultPort (urlToCheck) {
   const parsed = parseClone(urlToCheck)
 
   if (!parsed.port) {
     // unset host...
     // see above for reasoning
-    parsed.host = null
-    parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol]
+    parsed.host = undefined
+    parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol || 'http:']
   }
 
   return parsed
 }
 
-const getPath = (urlToCheck) => {
+export function getPath (urlToCheck) {
   return url.parse(urlToCheck).path
 }
-
-module.exports = {
-  parse,
-
-  getPath,
-
-  removePort,
-
-  addDefaultPort,
-
-  removeDefaultPort,
-
-  stripProtocolAndDefaultPorts,
-}

packages/network/package.json

@@ -12,6 +12,7 @@
     "bluebird": "3.5.3",
     "debug": "4.1.1",
     "lodash": "4.17.15",
+    "parse-domain": "2.0.0",
     "proxy-from-env": "1.0.0"
   },
   "devDependencies": {

packages/network/test/mocha.opts

@@ -1,5 +1,5 @@
 test/unit
 test/integration
---compilers ts:@packages/ts/register
+--compilers coffee:@packages/coffee/register,ts:@packages/ts/register
 --timeout 10000
 --recursive

packages/server/test/unit/blacklist_spec.coffee → packages/network/test/unit/blacklist_spec.coffee

@@ -1,6 +1,5 @@
-require("../spec_helper")
-
-blacklist = require("#{root}lib/util/blacklist")
+{ blacklist } = require("../..")
+{ expect } = require("chai")
 
 hosts = [
   "*.google.com"
@@ -21,7 +20,7 @@ matchesArray = (url, val) ->
 matchesHost = (url, host) ->
   expect(blacklist.matches(url, hosts)).to.eq(host)
 
-describe "lib/util/blacklist", ->
+describe "lib/blacklist", ->
   it "handles hosts, ports, wildcards", ->
     matchesArray("https://mail.google.com/foo", true)
     matchesArray("https://shop.apple.com/bar", true)

packages/server/test/unit/cors_spec.coffee → packages/network/test/unit/cors_spec.coffee

@@ -1,8 +1,7 @@
-require("../spec_helper")
+{ cors } = require("../..")
+{ expect } = require("chai")
 
-cors = require("#{root}lib/util/cors")
-
-describe "lib/util/cors", ->
+describe "lib/cors", ->
   context ".parseUrlIntoDomainTldPort", ->
     beforeEach ->
       @isEq = (url, obj) ->

packages/proxy/index.js

@@ -0,0 +1,5 @@
+if (process.env.CYPRESS_ENV !== 'production') {
+  require('@packages/ts/register')
+}
+
+module.exports = require('./lib')

packages/proxy/lib/http/error-middleware.ts

@@ -0,0 +1,41 @@
+import debugModule from 'debug'
+import { HttpMiddleware } from '.'
+import { Readable } from 'stream'
+import { Request } from 'request'
+
+const debug = debugModule('cypress:proxy:http:error-middleware')
+
+type ErrorMiddleware = HttpMiddleware<{
+  error: Error
+  incomingResStream?: Readable
+  outgoingReq?: Request
+}>
+
+export const AbortRequest : ErrorMiddleware = function () {
+  if (this.outgoingReq) {
+    debug('aborting outgoingReq')
+    this.outgoingReq.abort()
+  }
+
+  this.next()
+}
+
+export const UnpipeResponse : ErrorMiddleware = function () {
+  if (this.incomingResStream) {
+    debug('unpiping resStream from response')
+    this.incomingResStream.unpipe()
+  }
+
+  this.next()
+}
+
+export const DestroyResponse : ErrorMiddleware = function () {
+  this.res.destroy()
+  this.end()
+}
+
+export default {
+  AbortRequest,
+  UnpipeResponse,
+  DestroyResponse,
+}