diff --git a/circle.yml b/circle.yml index d1824b323ac9..e1c5a3a7dba2 100644 --- a/circle.yml +++ b/circle.yml @@ -262,6 +262,7 @@ jobs: - run: npm run all test -- --package https-proxy - run: npm run all test -- --package launcher - run: npm run all test -- --package network + - run: npm run all test -- --package proxy # how to pass Mocha reporter through zunder? - run: npm run all test -- --package reporter - run: npm run all test -- --package runner diff --git a/packages/network/lib/blacklist.ts b/packages/network/lib/blacklist.ts new file mode 100644 index 000000000000..1a6b7021ecba --- /dev/null +++ b/packages/network/lib/blacklist.ts @@ -0,0 +1,18 @@ +import _ from 'lodash' +import minimatch from 'minimatch' +import { stripProtocolAndDefaultPorts } from './uri' + +export function matches (urlToCheck, blacklistHosts) { + // normalize into flat array + blacklistHosts = [].concat(blacklistHosts) + + urlToCheck = stripProtocolAndDefaultPorts(urlToCheck) + + // use minimatch against the url + // to see if any match + const matchUrl = (hostMatcher) => { + return minimatch(urlToCheck, hostMatcher) + } + + return _.find(blacklistHosts, matchUrl) +} diff --git a/packages/network/lib/cors.ts b/packages/network/lib/cors.ts new file mode 100644 index 000000000000..546ddb3434f8 --- /dev/null +++ b/packages/network/lib/cors.ts @@ -0,0 +1,76 @@ +import _ from 'lodash' +import * as uri from './uri' +import debugModule from 'debug' +import parseDomain, { ParsedDomain } from 'parse-domain' + +const debug = debugModule('cypress:network:cors') + +const ipAddressRe = /^[\d\.]+$/ + +type ParsedHost = { + port?: string + tld?: string + domain?: string +} + +export function parseUrlIntoDomainTldPort (str) { + let { hostname, port, protocol } = uri.parse(str) + + if (!hostname) { + hostname = '' + } + + if (!port) { + port = protocol === 'https:' ? '443' : '80' + } + + let parsed : Partial | null = parseDomain(hostname, { + privateTlds: true, // use the public suffix + customTlds: ipAddressRe, + }) + + // if we couldn't get a parsed domain + if (!parsed) { + // then just fall back to a dumb check + // based on assumptions that the tld + // is the last segment after the final + // '.' and that the domain is the segment + // before that + const segments = hostname.split('.') + + parsed = { + tld: segments[segments.length - 1] || '', + domain: segments[segments.length - 2] || '', + } + } + + const obj: ParsedHost = {} + + obj.port = port + obj.tld = parsed.tld + obj.domain = parsed.domain + + debug('Parsed URL %o', obj) + + return obj +} + +export function urlMatchesOriginPolicyProps (urlStr, props) { + // take a shortcut here in the case + // where remoteHostAndPort is null + if (!props) { + return false + } + + const parsedUrl = parseUrlIntoDomainTldPort(urlStr) + + // does the parsedUrl match the parsedHost? + return _.isEqual(parsedUrl, props) +} + +export function urlMatchesOriginProtectionSpace (urlStr, origin) { + const normalizedUrl = uri.addDefaultPort(urlStr).format() + const normalizedOrigin = uri.addDefaultPort(origin).format() + + return _.startsWith(normalizedUrl, normalizedOrigin) +} diff --git a/packages/network/lib/index.ts b/packages/network/lib/index.ts index d2b58d7f42e8..52d425769bb1 100644 --- a/packages/network/lib/index.ts +++ b/packages/network/lib/index.ts @@ -1,9 +1,15 @@ import agent from './agent' -import * as connect from './connect' import { allowDestroy } from './allow-destroy' +import * as blacklist from './blacklist' +import * as connect from './connect' +import * as cors from './cors' +import * as uri from './uri' export { agent, allowDestroy, + blacklist, connect, + cors, + uri, } diff --git a/packages/server/lib/util/uri.js b/packages/network/lib/uri.ts similarity index 68% rename from packages/server/lib/util/uri.js rename to packages/network/lib/uri.ts index c5e59bff7b11..0255a952f353 100644 --- a/packages/server/lib/util/uri.js +++ b/packages/network/lib/uri.ts @@ -5,8 +5,8 @@ // node's url formatting algorithm (which acts pretty unexpectedly) // - https://nodejs.org/api/url.html#url_url_format_urlobject -const _ = require('lodash') -const url = require('url') +import _ from 'lodash' +import url from 'url' // yup, protocol contains a: ':' colon // at the end of it (-______________-) @@ -25,9 +25,9 @@ const parseClone = (urlObject) => { return url.parse(_.clone(urlObject)) } -const parse = url.parse +export const parse = url.parse -const stripProtocolAndDefaultPorts = function (urlToCheck) { +export function stripProtocolAndDefaultPorts (urlToCheck) { // grab host which is 'hostname:port' only const { host, hostname, port } = url.parse(urlToCheck) @@ -41,20 +41,18 @@ const stripProtocolAndDefaultPorts = function (urlToCheck) { return host } -const removePort = (urlObject) => { +export function removePort (urlObject) { const parsed = parseClone(urlObject) - // set host to null else - // url.format(...) will ignore - // the port property + // set host to undefined else url.format(...) will ignore the port property // https://nodejs.org/api/url.html#url_url_format_urlobject - parsed.host = null - parsed.port = null + parsed.host = undefined + parsed.port = undefined return parsed } -const removeDefaultPort = function (urlToCheck) { +export function removeDefaultPort (urlToCheck) { let parsed = parseClone(urlToCheck) if (portIsDefault(parsed.port)) { @@ -64,33 +62,19 @@ const removeDefaultPort = function (urlToCheck) { return parsed } -const addDefaultPort = function (urlToCheck) { +export function addDefaultPort (urlToCheck) { const parsed = parseClone(urlToCheck) if (!parsed.port) { // unset host... // see above for reasoning - parsed.host = null - parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol] + parsed.host = undefined + parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol || 'http:'] } return parsed } -const getPath = (urlToCheck) => { +export function getPath (urlToCheck) { return url.parse(urlToCheck).path } - -module.exports = { - parse, - - getPath, - - removePort, - - addDefaultPort, - - removeDefaultPort, - - stripProtocolAndDefaultPorts, -} diff --git a/packages/network/package.json b/packages/network/package.json index 9e58d0ed60cd..c6dce437339e 100644 --- a/packages/network/package.json +++ b/packages/network/package.json @@ -12,6 +12,7 @@ "bluebird": "3.5.3", "debug": "4.1.1", "lodash": "4.17.15", + "parse-domain": "2.0.0", "proxy-from-env": "1.0.0" }, "devDependencies": { diff --git a/packages/network/test/mocha.opts b/packages/network/test/mocha.opts index 7ef9f2476490..570cc398a73b 100644 --- a/packages/network/test/mocha.opts +++ b/packages/network/test/mocha.opts @@ -1,5 +1,5 @@ test/unit test/integration ---compilers ts:@packages/ts/register +--compilers coffee:@packages/coffee/register,ts:@packages/ts/register --timeout 10000 --recursive diff --git a/packages/server/test/unit/blacklist_spec.coffee b/packages/network/test/unit/blacklist_spec.coffee similarity index 92% rename from packages/server/test/unit/blacklist_spec.coffee rename to packages/network/test/unit/blacklist_spec.coffee index f0796cea355d..34965d43520b 100644 --- a/packages/server/test/unit/blacklist_spec.coffee +++ b/packages/network/test/unit/blacklist_spec.coffee @@ -1,6 +1,5 @@ -require("../spec_helper") - -blacklist = require("#{root}lib/util/blacklist") +{ blacklist } = require("../..") +{ expect } = require("chai") hosts = [ "*.google.com" @@ -21,7 +20,7 @@ matchesArray = (url, val) -> matchesHost = (url, host) -> expect(blacklist.matches(url, hosts)).to.eq(host) -describe "lib/util/blacklist", -> +describe "lib/blacklist", -> it "handles hosts, ports, wildcards", -> matchesArray("https://mail.google.com/foo", true) matchesArray("https://shop.apple.com/bar", true) diff --git a/packages/server/test/unit/cors_spec.coffee b/packages/network/test/unit/cors_spec.coffee similarity index 98% rename from packages/server/test/unit/cors_spec.coffee rename to packages/network/test/unit/cors_spec.coffee index 96ef40eaee7e..dd75289696ad 100644 --- a/packages/server/test/unit/cors_spec.coffee +++ b/packages/network/test/unit/cors_spec.coffee @@ -1,8 +1,7 @@ -require("../spec_helper") +{ cors } = require("../..") +{ expect } = require("chai") -cors = require("#{root}lib/util/cors") - -describe "lib/util/cors", -> +describe "lib/cors", -> context ".parseUrlIntoDomainTldPort", -> beforeEach -> @isEq = (url, obj) -> diff --git a/packages/proxy/index.js b/packages/proxy/index.js new file mode 100644 index 000000000000..99166f024f49 --- /dev/null +++ b/packages/proxy/index.js @@ -0,0 +1,5 @@ +if (process.env.CYPRESS_ENV !== 'production') { + require('@packages/ts/register') +} + +module.exports = require('./lib') diff --git a/packages/proxy/lib/http/error-middleware.ts b/packages/proxy/lib/http/error-middleware.ts new file mode 100644 index 000000000000..1cf4d434b4df --- /dev/null +++ b/packages/proxy/lib/http/error-middleware.ts @@ -0,0 +1,41 @@ +import debugModule from 'debug' +import { HttpMiddleware } from '.' +import { Readable } from 'stream' +import { Request } from 'request' + +const debug = debugModule('cypress:proxy:http:error-middleware') + +type ErrorMiddleware = HttpMiddleware<{ + error: Error + incomingResStream?: Readable + outgoingReq?: Request +}> + +export const AbortRequest : ErrorMiddleware = function () { + if (this.outgoingReq) { + debug('aborting outgoingReq') + this.outgoingReq.abort() + } + + this.next() +} + +export const UnpipeResponse : ErrorMiddleware = function () { + if (this.incomingResStream) { + debug('unpiping resStream from response') + this.incomingResStream.unpipe() + } + + this.next() +} + +export const DestroyResponse : ErrorMiddleware = function () { + this.res.destroy() + this.end() +} + +export default { + AbortRequest, + UnpipeResponse, + DestroyResponse, +} diff --git a/packages/proxy/lib/http/index.ts b/packages/proxy/lib/http/index.ts new file mode 100644 index 000000000000..b0aef2a33c0a --- /dev/null +++ b/packages/proxy/lib/http/index.ts @@ -0,0 +1,230 @@ +import _ from 'lodash' +import debugModule from 'debug' +import ErrorMiddleware from './error-middleware' +import { HttpBuffers } from './util/buffers' +import { IncomingMessage } from 'http' +import Promise from 'bluebird' +import { Readable } from 'stream' +import { Request, Response } from 'express' +import RequestMiddleware from './request-middleware' +import ResponseMiddleware from './response-middleware' + +const debug = debugModule('cypress:proxy:http') + +export enum HttpStages { + IncomingRequest, + IncomingResponse, + Error +} + +export type HttpMiddleware = (this: HttpMiddlewareThis) => void + +export type CypressRequest = Request & { + // TODO: what's this difference from req.url? is it only for non-proxied requests? + proxiedUrl: string + abort: () => void +} + +type MiddlewareStacks = { + [stage in HttpStages]: { + [name: string]: HttpMiddleware + } +} + +export type CypressResponse = Response & { + isInitial: null | boolean + wantsInjection: 'full' | 'partial' | false + wantsSecurityRemoved: null | boolean +} + +type HttpMiddlewareCtx = { + req: CypressRequest + res: CypressResponse + + middleware: MiddlewareStacks +} & T + +const READONLY_MIDDLEWARE_KEYS : (keyof HttpMiddlewareThis<{}>)[] = [ + 'buffers', + 'config', + 'getRemoteState', + 'next', + 'end', + 'onResponse', + 'onError', + 'skipMiddleware', +] + +type HttpMiddlewareThis = HttpMiddlewareCtx & Readonly<{ + buffers: HttpBuffers + config: any + getRemoteState: () => any + + next: () => void + /** + * Call to completely end the stage, bypassing any remaining middleware. + */ + end: () => void + onResponse: (incomingRes: Response, resStream: Readable) => void + onError: (error: Error) => void + skipMiddleware: (name: string) => void +}> + +export function _runStage (type: HttpStages, ctx: any) { + const stage = HttpStages[type] + + debug('Entering stage %o', { stage }) + + const runMiddlewareStack = () => { + const middlewares = ctx.middleware[type] + + // pop the first pair off the middleware + const middlewareName = _.keys(middlewares)[0] + + if (!middlewareName) { + return Promise.resolve() + } + + const middleware = middlewares[middlewareName] + + ctx.middleware[type] = _.omit(middlewares, middlewareName) + + return new Promise((resolve) => { + let ended = false + + function copyChangedCtx () { + _.chain(fullCtx) + .omit(READONLY_MIDDLEWARE_KEYS) + .forEach((value, key) => { + if (ctx[key] !== value) { + debug(`copying %o`, { [key]: value }) + ctx[key] = value + } + }) + .value() + } + + function _end (retval?) { + if (ended) { + return + } + + ended = true + + copyChangedCtx() + + resolve(retval) + } + + if (!middleware) { + return resolve() + } + + debug('Running middleware %o', { stage, middlewareName }) + + const fullCtx = { + next: () => { + copyChangedCtx() + + _end(runMiddlewareStack()) + }, + end: () => _end(), + onResponse: (incomingRes: IncomingMessage, resStream: Readable) => { + ctx.incomingRes = incomingRes + ctx.incomingResStream = resStream + + _end() + }, + onError: (error: Error) => { + debug('Error in middleware %o', { stage, middlewareName, error, ctx }) + + if (type === HttpStages.Error) { + return + } + + ctx.error = error + + _end(_runStage(HttpStages.Error, ctx)) + }, + + skipMiddleware: (name) => { + ctx.middleware[type] = _.omit(ctx.middleware[type], name) + }, + + ...ctx, + } + + try { + middleware.call(fullCtx) + } catch (err) { + fullCtx.onError(err) + } + }) + } + + return runMiddlewareStack() + .then(() => { + debug('Leaving stage %o', { stage }) + }) +} + +export class Http { + buffers: HttpBuffers + config: any + getRemoteState: () => any + middleware: MiddlewareStacks + + constructor (opts: { + config: any + getRemoteState: () => any + middleware?: MiddlewareStacks + }) { + this.buffers = new HttpBuffers() + + this.config = opts.config + this.getRemoteState = opts.getRemoteState + + if (typeof opts.middleware === 'undefined') { + this.middleware = { + [HttpStages.IncomingRequest]: RequestMiddleware, + [HttpStages.IncomingResponse]: ResponseMiddleware, + [HttpStages.Error]: ErrorMiddleware, + } + } else { + this.middleware = opts.middleware + } + } + + handle (req, res) { + const ctx : HttpMiddlewareCtx = { + req, + res, + + buffers: this.buffers, + config: this.config, + getRemoteState: this.getRemoteState, + middleware: _.cloneDeep(this.middleware), + } + + return _runStage(HttpStages.IncomingRequest, ctx) + .then(() => { + if (ctx.incomingRes) { + return _runStage(HttpStages.IncomingResponse, ctx) + } + + return debug('warning: Request was not fulfilled with a response.') + }) + } + + reset () { + this.buffers.reset() + } + + getBuffer (urlStr) { + return this.buffers.getByOriginalUrl(urlStr) + } + + setBuffer (buffer) { + return this.buffers.set(buffer) + } +} diff --git a/packages/proxy/lib/http/request-middleware.ts b/packages/proxy/lib/http/request-middleware.ts new file mode 100644 index 000000000000..fe652a83af42 --- /dev/null +++ b/packages/proxy/lib/http/request-middleware.ts @@ -0,0 +1,151 @@ +import { blacklist, cors } from '@packages/network' +import debugModule from 'debug' +import { HttpMiddleware } from '.' + +type RequestMiddleware = HttpMiddleware<{ + outgoingReq: any +}> + +const Request = require('../../../server/lib/request') + +const debug = debugModule('cypress:proxy:http:request-middleware') + +// TODO: refactor +const request = Request() + +const RedirectToClientRouteIfUnloaded : RequestMiddleware = function () { + // if we have an unload header it means our parent app has been navigated away + // directly and we need to automatically redirect to the clientRoute + if (this.req.cookies['__cypress.unload']) { + this.res.redirect(this.config.clientRoute) + + return this.end() + } + + this.next() +} + +// TODO: is this necessary? it seems to be for requesting Cypress w/o the proxy, +// which isn't currently supported +const RedirectToClientRouteIfNotProxied : RequestMiddleware = function () { + // when you access cypress from a browser which has not had its proxy setup then + // req.url will match req.proxiedUrl and we'll know to instantly redirect them + // to the correct client route + if (this.req.url === this.req.proxiedUrl && !this.getRemoteState().visiting) { + // if we dont have a remoteState.origin that means we're initially requesting + // the cypress app and we need to redirect to the root path that serves the app + this.res.redirect(this.config.clientRoute) + + return this.end() + } + + this.next() +} + +const EndRequestsToBlacklistedHosts : RequestMiddleware = function () { + const { blacklistHosts } = this.config + + if (blacklistHosts) { + const matches = blacklist.matches(this.req.proxiedUrl, blacklistHosts) + + if (matches) { + this.res.set('x-cypress-matched-blacklisted-host', matches) + debug('blacklisting request %o', { + url: this.req.proxiedUrl, + matches, + }) + + this.res.status(503).end() + + return this.end() + } + } + + this.next() +} + +const MaybeEndRequestWithBufferedResponse : RequestMiddleware = function () { + const buffer = this.buffers.take(this.req.proxiedUrl) + + if (buffer) { + debug('Got a buffer') + this.res.wantsInjection = 'full' + + return this.onResponse(buffer.response, buffer.stream) + } + + this.next() +} + +const StripUnsupportedAcceptEncoding : RequestMiddleware = function () { + // Cypress can only support plaintext or gzip, so make sure we don't request anything else + const acceptEncoding = this.req.headers['accept-encoding'] + + if (acceptEncoding) { + if (acceptEncoding.includes('gzip')) { + this.req.headers['accept-encoding'] = 'gzip' + } else { + delete this.req.headers['accept-encoding'] + } + } + + this.next() +} + +function reqNeedsBasicAuthHeaders (req, { auth, origin }) { + //if we have auth headers, this request matches our origin, protection space, and the user has not supplied auth headers + return auth && !req.headers['authorization'] && cors.urlMatchesOriginProtectionSpace(req.proxiedUrl, origin) +} + +const MaybeSetBasicAuthHeaders : RequestMiddleware = function () { + const remoteState = this.getRemoteState() + + if (reqNeedsBasicAuthHeaders(this.req, remoteState)) { + const { auth } = remoteState + const base64 = Buffer.from(`${auth.username}:${auth.password}`).toString('base64') + + this.req.headers['authorization'] = `Basic ${base64}` + } + + this.next() +} + +const SendRequestOutgoing : RequestMiddleware = function () { + const requestOptions = { + timeout: this.config.responseTimeout, + strictSSL: false, + followRedirect: false, + retryIntervals: [0, 100, 200, 200], + url: this.req.proxiedUrl, + } + + const remoteState = this.getRemoteState() + + if (remoteState.strategy === 'file' && requestOptions.url.startsWith(remoteState.origin)) { + requestOptions.url = requestOptions.url.replace(remoteState.origin, remoteState.fileServer) + } + + const req = request.create(requestOptions) + + req.on('error', this.onError) + req.on('response', (incomingRes) => this.onResponse(incomingRes, req)) + this.req.on('aborted', () => { + debug('request aborted') + req.abort() + }) + + // pipe incoming request body, headers to new request + this.req.pipe(req) + + this.outgoingReq = req +} + +export default { + RedirectToClientRouteIfUnloaded, + RedirectToClientRouteIfNotProxied, + EndRequestsToBlacklistedHosts, + MaybeEndRequestWithBufferedResponse, + StripUnsupportedAcceptEncoding, + MaybeSetBasicAuthHeaders, + SendRequestOutgoing, +} diff --git a/packages/proxy/lib/http/response-middleware.ts b/packages/proxy/lib/http/response-middleware.ts new file mode 100644 index 000000000000..fe405d4661d9 --- /dev/null +++ b/packages/proxy/lib/http/response-middleware.ts @@ -0,0 +1,333 @@ +import _ from 'lodash' +import charset from 'charset' +import concatStream from 'concat-stream' +import { CookieOptions } from 'express' +import { cors } from '@packages/network' +import { CypressRequest, CypressResponse, HttpMiddleware } from '.' +import debugModule from 'debug' +import iconv from 'iconv-lite' +import { IncomingMessage, IncomingHttpHeaders } from 'http' +import { PassThrough, Readable } from 'stream' +import * as rewriter from './util/rewriter' +import zlib from 'zlib' + +export type ResponseMiddleware = HttpMiddleware<{ + incomingRes: IncomingMessage + incomingResStream: Readable +}> + +const debug = debugModule('cypress:proxy:http:response-middleware') + +// https://github.com/cypress-io/cypress/issues/1756 +const zlibOptions = { + flush: zlib.Z_SYNC_FLUSH, + finishFlush: zlib.Z_SYNC_FLUSH, +} + +// https://github.com/cypress-io/cypress/issues/1543 +function getNodeCharsetFromResponse (headers: IncomingHttpHeaders, body: Buffer) { + const httpCharset = (charset(headers, body, 1024) || '').toLowerCase() + + debug('inferred charset from response %o', { httpCharset }) + if (iconv.encodingExists(httpCharset)) { + return httpCharset + } + + // browsers default to latin1 + return 'latin1' +} + +function reqMatchesOriginPolicy (req: CypressRequest, remoteState) { + if (remoteState.strategy === 'http') { + return cors.urlMatchesOriginPolicyProps(req.proxiedUrl, remoteState.props) + } + + if (remoteState.strategy === 'file') { + return req.proxiedUrl.startsWith(remoteState.origin) + } + + return false +} + +function reqWillRenderHtml (req: CypressRequest) { + // will this request be rendered in the browser, necessitating injection? + // https://github.com/cypress-io/cypress/issues/288 + + // don't inject if this is an XHR from jquery + if (req.headers['x-requested-with']) { + return + } + + // don't inject if we didn't find both text/html and application/xhtml+xml, + const accept = req.headers['accept'] + + return accept && accept.includes('text/html') && accept.includes('application/xhtml+xml') +} + +function resContentTypeIs (res: IncomingMessage, contentType: string) { + return (res.headers['content-type'] || '').includes(contentType) +} + +function resContentTypeIsJavaScript (res: IncomingMessage) { + return _.some( + ['application/javascript', 'application/x-javascript', 'text/javascript'] + .map(_.partial(resContentTypeIs, res)) + ) +} + +function resIsGzipped (res: IncomingMessage) { + return (res.headers['content-encoding'] || '').includes('gzip') +} + +// https://github.com/cypress-io/cypress/issues/4298 +// https://tools.ietf.org/html/rfc7230#section-3.3.3 +// HEAD, 1xx, 204, and 304 responses should never contain anything after headers +const NO_BODY_STATUS_CODES = [204, 304] + +function responseMustHaveEmptyBody (req: CypressRequest, res: IncomingMessage) { + return _.some([_.includes(NO_BODY_STATUS_CODES, res.statusCode), _.invoke(req.method, 'toLowerCase') === 'head']) +} + +function setCookie (res: CypressResponse, k: string, v: string, domain: string) { + let opts : CookieOptions = { domain } + + if (!v) { + v = '' + + opts.expires = new Date(0) + } + + return res.cookie(k, v, opts) +} + +function setInitialCookie (res: CypressResponse, remoteState: any, value) { + // dont modify any cookies if we're trying to clear the initial cookie and we're not injecting anything + // dont set the cookies if we're not on the initial request + if ((!value && !res.wantsInjection) || !res.isInitial) { + return + } + + return setCookie(res, '__cypress.initial', value, remoteState.domainName) +} + +const Log : ResponseMiddleware = function () { + debug('received response %o', { + req: _.pick(this.req, 'method', 'proxiedUrl', 'headers'), + incomingRes: _.pick(this.incomingRes, 'headers', 'statusCode'), + }) + + this.next() +} + +const PatchExpressSetHeader : ResponseMiddleware = function () { + const originalSetHeader = this.res.setHeader + + // express.Response.setHeader does all kinds of silly/nasty stuff to the content-type... + // but we don't want to change it at all! + this.res.setHeader = (k, v) => { + if (k === 'content-type') { + v = this.incomingRes.headers['content-type'] || v + } + + return originalSetHeader.call(this.res, k, v) + } + + this.next() +} + +const SetInjectionLevel : ResponseMiddleware = function () { + this.res.isInitial = this.req.cookies['__cypress.initial'] === 'true' + + const getInjectionLevel = () => { + if (this.incomingRes.headers['x-cypress-file-server-error'] && !this.res.isInitial) { + return 'partial' + } + + if (!resContentTypeIs(this.incomingRes, 'text/html') || !reqMatchesOriginPolicy(this.req, this.getRemoteState())) { + return false + } + + if (this.res.isInitial) { + return 'full' + } + + if (!reqWillRenderHtml(this.req)) { + return false + } + + return 'partial' + } + + if (!this.res.wantsInjection) { + this.res.wantsInjection = getInjectionLevel() + } + + this.res.wantsSecurityRemoved = this.config.modifyObstructiveCode && ( + (this.res.wantsInjection === 'full') + || resContentTypeIsJavaScript(this.incomingRes) + ) + + debug('injection levels: %o', _.pick(this.res, 'isInitial', 'wantsInjection', 'wantsSecurityRemoved')) + + this.next() +} + +const OmitProblematicHeaders : ResponseMiddleware = function () { + const headers = _.omit(this.incomingRes.headers, [ + 'set-cookie', + 'x-frame-options', + 'content-length', + 'content-security-policy', + 'connection', + ]) + + this.res.set(headers) + + this.next() +} + +const MaybePreventCaching : ResponseMiddleware = function () { + // do not cache injected responses + // TODO: consider implementing etag system so even injected content can be cached + if (this.res.wantsInjection) { + this.res.setHeader('cache-control', 'no-cache, no-store, must-revalidate') + } + + this.next() +} + +const CopyCookiesFromIncomingRes : ResponseMiddleware = function () { + const cookies : string | string[] | undefined = this.incomingRes.headers['set-cookie'] + + if (cookies) { + ([] as string[]).concat(cookies).forEach((cookie) => { + try { + this.res.append('Set-Cookie', cookie) + } catch (e) { + debug('failed to Set-Cookie, continuing', { cookie }) + } + }) + } + + this.next() +} + +const REDIRECT_STATUS_CODES : any[] = [301, 302, 303, 307, 308] + +// TODO: this shouldn't really even be necessary? +const MaybeSendRedirectToClient : ResponseMiddleware = function () { + const { statusCode, headers } = this.incomingRes + const newUrl = headers['location'] + + if (!REDIRECT_STATUS_CODES.includes(statusCode) || !newUrl) { + return this.next() + } + + setInitialCookie(this.res, this.getRemoteState(), true) + + debug('redirecting to new url %o', { statusCode, newUrl }) + this.res.redirect(Number(statusCode), newUrl) + + return this.end() +} + +const CopyResponseStatusCode : ResponseMiddleware = function () { + this.res.status(Number(this.incomingRes.statusCode)) + this.next() +} + +const ClearCyInitialCookie : ResponseMiddleware = function () { + setInitialCookie(this.res, this.getRemoteState(), false) + this.next() +} + +const MaybeEndWithEmptyBody : ResponseMiddleware = function () { + if (responseMustHaveEmptyBody(this.req, this.incomingRes)) { + this.res.end() + + return this.end() + } + + this.next() +} + +const MaybeGunzipBody : ResponseMiddleware = function () { + if (resIsGzipped(this.incomingRes) && (this.res.wantsInjection || this.res.wantsSecurityRemoved)) { + debug('ungzipping response body') + + const gunzip = zlib.createGunzip(zlibOptions) + + this.incomingResStream = this.incomingResStream.pipe(gunzip).on('error', this.onError) + } else { + this.skipMiddleware('GzipBody') // not needed anymore + } + + this.next() +} + +const MaybeInjectHtml : ResponseMiddleware = function () { + if (!this.res.wantsInjection) { + return this.next() + } + + this.skipMiddleware('MaybeRemoveSecurity') // we only want to do one or the other + + debug('injecting into HTML') + + this.incomingResStream.pipe(concatStream((body) => { + const nodeCharset = getNodeCharsetFromResponse(this.incomingRes.headers, body) + const decodedBody = iconv.decode(body, nodeCharset) + const injectedBody = rewriter.html(decodedBody, this.getRemoteState().domainName, this.res.wantsInjection, this.res.wantsSecurityRemoved) + const encodedBody = iconv.encode(injectedBody, nodeCharset) + + const pt = new PassThrough + + pt.write(encodedBody) + pt.end() + + this.incomingResStream = pt + this.next() + })).on('error', this.onError) +} + +const MaybeRemoveSecurity : ResponseMiddleware = function () { + if (!this.res.wantsSecurityRemoved) { + return this.next() + } + + debug('removing JS framebusting code') + + this.incomingResStream.setEncoding('utf8') + this.incomingResStream = this.incomingResStream.pipe(rewriter.security()).on('error', this.onError) + this.next() +} + +const GzipBody : ResponseMiddleware = function () { + debug('regzipping response body') + this.incomingResStream = this.incomingResStream.pipe(zlib.createGzip(zlibOptions)).on('error', this.onError) + + this.next() +} + +const SendResponseBodyToClient : ResponseMiddleware = function () { + this.incomingResStream.pipe(this.res).on('error', this.onError) + this.res.on('end', () => this.end()) +} + +export default { + Log, + PatchExpressSetHeader, + SetInjectionLevel, + OmitProblematicHeaders, + MaybePreventCaching, + CopyCookiesFromIncomingRes, + MaybeSendRedirectToClient, + CopyResponseStatusCode, + ClearCyInitialCookie, + MaybeEndWithEmptyBody, + MaybeGunzipBody, + MaybeInjectHtml, + MaybeRemoveSecurity, + GzipBody, + SendResponseBodyToClient, +} diff --git a/packages/proxy/lib/http/util/buffers.ts b/packages/proxy/lib/http/util/buffers.ts new file mode 100644 index 000000000000..9101fa22003c --- /dev/null +++ b/packages/proxy/lib/http/util/buffers.ts @@ -0,0 +1,80 @@ +import _ from 'lodash' +import debugModule from 'debug' +import { uri } from '@packages/network' +import { Readable } from 'stream' +import { Response } from 'express' + +const debug = debugModule('cypress:proxy:http:util:buffers') + +type HttpBuffer = { + details: object + jar: any + originalUrl: string + response: Response + stream: Readable + url: string +} + +export class HttpBuffers { + buffers : HttpBuffer[] = [] + + all () { + return this.buffers + } + + keys () { + return _.map(this.buffers, 'url') + } + + reset () { + debug('resetting buffers') + + this.buffers = [] + } + + set (obj = {}) { + return this.buffers.push(_.pick(obj, 'url', 'originalUrl', 'jar', 'stream', 'response', 'details') as HttpBuffer) + } + + getByOriginalUrl (str) { + return _.find(this.buffers, { originalUrl: str }) + } + + get (str) { + const find = (str) => { + return _.find(this.buffers, { url: str }) + } + + const b = find(str) + + if (b) { + return b + } + + let parsed = uri.parse(str) + + // if we're on https and we have a port + // then attempt to find the buffer by + // slicing off the port since our buffer + // was likely stored without a port + if ((parsed.protocol === 'https:') && parsed.port) { + parsed = uri.removePort(parsed) + + return find(parsed.format()) + } + + return undefined + } + + take (str) { + const buffer = this.get(str) + + if (buffer) { + this.buffers = _.without(this.buffers, buffer) + + debug('found request buffer %o', { buffer: _.pick(buffer, 'url', 'originalUrl'), bufferCount: this.buffers.length }) + } + + return buffer + } +} diff --git a/packages/proxy/lib/http/util/inject.ts b/packages/proxy/lib/http/util/inject.ts new file mode 100644 index 000000000000..e84cb731142d --- /dev/null +++ b/packages/proxy/lib/http/util/inject.ts @@ -0,0 +1,25 @@ +const { oneLine } = require('common-tags') + +export function partial (domain) { + return oneLine` + + ` +} + +export function full (domain) { + return oneLine` + + ` +} diff --git a/packages/server/lib/util/rewriter.js b/packages/proxy/lib/http/util/rewriter.ts similarity index 58% rename from packages/server/lib/util/rewriter.js rename to packages/proxy/lib/http/util/rewriter.ts index ec675036f743..4829017d0a45 100644 --- a/packages/server/lib/util/rewriter.js +++ b/packages/proxy/lib/http/util/rewriter.ts @@ -1,22 +1,12 @@ -/* eslint-disable - default-case, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const inject = require('./inject') -const security = require('./security') +import * as inject from './inject' +import { strip, stripStream } from './security' const doctypeRe = /(<\!doctype.*?>)/i const headRe = /()/i const bodyRe = /()/i const htmlRe = /()/i -const rewriteHtml = function (html, domainName, wantsInjection, wantsSecurityRemoved) { +export function html (html: string, domainName: string, wantsInjection, wantsSecurityRemoved) { const replace = (re, str) => { return html.replace(re, str) } @@ -27,13 +17,15 @@ const rewriteHtml = function (html, domainName, wantsInjection, wantsSecurityRem return inject.full(domainName) case 'partial': return inject.partial(domainName) + default: + return } })() - //# strip clickjacking and framebusting - //# from the HTML if we've been told to + // strip clickjacking and framebusting + // from the HTML if we've been told to if (wantsSecurityRemoved) { - html = security.strip(html) + html = strip(html) } switch (false) { @@ -55,8 +47,4 @@ const rewriteHtml = function (html, domainName, wantsInjection, wantsSecurityRem } } -module.exports = { - html: rewriteHtml, - - security: security.stripStream, -} +export const security = stripStream diff --git a/packages/server/lib/util/security.js b/packages/proxy/lib/http/util/security.ts similarity index 86% rename from packages/server/lib/util/security.js rename to packages/proxy/lib/http/util/security.ts index 10566960209b..83f529e84371 100644 --- a/packages/server/lib/util/security.js +++ b/packages/proxy/lib/http/util/security.ts @@ -1,5 +1,3 @@ -// Tests located in packages/server/test/unit/security_spec - const pumpify = require('pumpify') const replacestream = require('replacestream') @@ -8,7 +6,7 @@ const topOrParentEqualityAfterRe = /(top|parent)((?:["']\])?\s*[!=]==?\s*(?:wind const topOrParentLocationOrFramesRe = /([^\da-zA-Z\(\)])?(top|parent)([.])(location|frames)/g const jiraTopWindowGetterRe = /(!function\s*\((\w{1})\)\s*{\s*return\s*\w{1}\s*(?:={2,})\s*\w{1}\.parent)(\s*}\(\w{1}\))/g -const strip = (html) => { +export function strip (html: string) { return html .replace(topOrParentEqualityBeforeRe, '$1self') .replace(topOrParentEqualityAfterRe, 'self$2') @@ -16,7 +14,7 @@ const strip = (html) => { .replace(jiraTopWindowGetterRe, '$1 || $2.parent.__Cypress__$3') } -const stripStream = () => { +export function stripStream () { return pumpify( replacestream(topOrParentEqualityBeforeRe, '$1self'), replacestream(topOrParentEqualityAfterRe, 'self$2'), @@ -24,9 +22,3 @@ const stripStream = () => { replacestream(jiraTopWindowGetterRe, '$1 || $2.parent.__Cypress__$3') ) } - -module.exports = { - strip, - - stripStream, -} diff --git a/packages/proxy/lib/index.ts b/packages/proxy/lib/index.ts new file mode 100644 index 000000000000..5361f3f8aaab --- /dev/null +++ b/packages/proxy/lib/index.ts @@ -0,0 +1 @@ +export { NetworkProxy } from './network-proxy' diff --git a/packages/proxy/lib/network-proxy.ts b/packages/proxy/lib/network-proxy.ts new file mode 100644 index 000000000000..eb81a9798a76 --- /dev/null +++ b/packages/proxy/lib/network-proxy.ts @@ -0,0 +1,36 @@ +import { Http } from './http' + +export class NetworkProxy { + http: Http + + constructor (opts: { + config: any + getRemoteState: () => {} + }) { + this.http = new Http(opts) + } + + handleHttpRequest (req, res) { + this.http.handle(req, res) + } + + handleHttpsConnect () { + + } + + handleWsUpgrade () { + + } + + getHttpBuffer (urlStr) { + return this.http.getBuffer(urlStr) + } + + setHttpBuffer (buffer) { + this.http.setBuffer(buffer) + } + + reset () { + this.http.reset() + } +} diff --git a/packages/proxy/package.json b/packages/proxy/package.json new file mode 100644 index 000000000000..199f1764d77e --- /dev/null +++ b/packages/proxy/package.json @@ -0,0 +1,31 @@ +{ + "name": "@packages/proxy", + "version": "0.0.0", + "private": true, + "main": "index.js", + "scripts": { + "build-js": "bin-up tsc --project .", + "clean-deps": "rm -rf node_modules", + "test": "bin-up mocha --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json" + }, + "dependencies": { + "bluebird": "3.5.3", + "charset": "^1.0.1", + "concat-stream": "^1.6.2", + "debug": "4.1.1", + "iconv-lite": "^0.5.0", + "lodash": "4.17.15", + "replacestream": "^4.0.3" + }, + "devDependencies": { + "@cypress/sinon-chai": "^2.9.0", + "@types/express": "^4.17.1", + "bin-up": "1.2.0", + "request": "^2.88.0", + "request-promise": "^4.2.4" + }, + "files": [ + "lib" + ], + "types": "./lib/index.ts" +} diff --git a/packages/proxy/test/.eslintrc b/packages/proxy/test/.eslintrc new file mode 100644 index 000000000000..b5ed5206d083 --- /dev/null +++ b/packages/proxy/test/.eslintrc @@ -0,0 +1,5 @@ +{ + "extends": [ + "plugin:@cypress/dev/tests" + ] +} diff --git a/packages/proxy/test/mocha.opts b/packages/proxy/test/mocha.opts new file mode 100644 index 000000000000..7a20898f8d83 --- /dev/null +++ b/packages/proxy/test/mocha.opts @@ -0,0 +1,5 @@ +test/unit +--compilers ts:@packages/ts/register,coffee:@packages/coffee/register +--timeout 10000 +--recursive +--require test/pretest.ts diff --git a/packages/proxy/test/pretest.ts b/packages/proxy/test/pretest.ts new file mode 100644 index 000000000000..471760d6c14d --- /dev/null +++ b/packages/proxy/test/pretest.ts @@ -0,0 +1,2 @@ +require('chai') +.use(require('@cypress/sinon-chai')) diff --git a/packages/proxy/test/unit/http/error-middleware.spec.ts b/packages/proxy/test/unit/http/error-middleware.spec.ts new file mode 100644 index 000000000000..46ded1073ce3 --- /dev/null +++ b/packages/proxy/test/unit/http/error-middleware.spec.ts @@ -0,0 +1,74 @@ +import _ from 'lodash' +import ErrorMiddleware, { + AbortRequest, + UnpipeResponse, + DestroyResponse, +} from '../../../lib/http/error-middleware' +import { expect } from 'chai' +import sinon from 'sinon' +import { + testMiddleware, +} from './helpers' + +describe('http/error-middleware', function () { + it('exports the members in the correct order', function () { + expect(_.keys(ErrorMiddleware)).to.have.ordered.members([ + 'AbortRequest', + 'UnpipeResponse', + 'DestroyResponse', + ]) + }) + + context('AbortRequest', function () { + it('destroys outgoingReq if it exists', function () { + const ctx = { + outgoingReq: { + abort: sinon.stub(), + }, + } + + return testMiddleware([AbortRequest], ctx) + .then(() => { + expect(ctx.outgoingReq.abort).to.be.calledOnce + }) + }) + + it('does not destroy outgoingReq if it does not exist', function () { + return testMiddleware([AbortRequest], {}) + }) + }) + + context('UnpipeResponse', function () { + it('unpipes incomingRes if it exists', function () { + const ctx = { + incomingResStream: { + unpipe: sinon.stub(), + }, + } + + return testMiddleware([UnpipeResponse], ctx) + .then(() => { + expect(ctx.incomingResStream.unpipe).to.be.calledOnce + }) + }) + + it('does not unpipe incomingRes if it does not exist', function () { + return testMiddleware([UnpipeResponse], {}) + }) + }) + + context('DestroyResponse', function () { + it('destroys the response', function () { + const ctx = { + res: { + destroy: sinon.stub(), + }, + } + + return testMiddleware([DestroyResponse], ctx) + .then(() => { + expect(ctx.res.destroy).to.be.calledOnce + }) + }) + }) +}) diff --git a/packages/proxy/test/unit/http/helpers.ts b/packages/proxy/test/unit/http/helpers.ts new file mode 100644 index 000000000000..70b1720fdbf5 --- /dev/null +++ b/packages/proxy/test/unit/http/helpers.ts @@ -0,0 +1,18 @@ +import { HttpMiddleware, _runStage } from '../../../lib/http' + +export function testMiddleware (middleware: HttpMiddleware[], ctx = {}) { + const fullCtx = { + req: {}, + res: {}, + config: {}, + getRemoteState: () => {}, + + middleware: { + 0: middleware, + }, + + ...ctx, + } + + return _runStage(0, fullCtx) +} diff --git a/packages/proxy/test/unit/http/index.spec.ts b/packages/proxy/test/unit/http/index.spec.ts new file mode 100644 index 000000000000..e8e29d3988d2 --- /dev/null +++ b/packages/proxy/test/unit/http/index.spec.ts @@ -0,0 +1,160 @@ +import { Http, HttpStages } from '../../../lib/http' +import { expect } from 'chai' +import sinon from 'sinon' + +describe('http', function () { + context('Http.handle', function () { + let config + let getRemoteState + let middleware + let incomingRequest + let incomingResponse + let error + + beforeEach(function () { + config = {} + getRemoteState = sinon.stub().returns({}) + + incomingRequest = sinon.stub() + incomingResponse = sinon.stub() + error = sinon.stub() + + middleware = { + [HttpStages.IncomingRequest]: [incomingRequest], + [HttpStages.IncomingResponse]: [incomingResponse], + [HttpStages.Error]: [error], + } + }) + + it('calls IncomingRequest stack, then IncomingResponse stack', function () { + incomingRequest.callsFake(function () { + expect(incomingResponse).to.not.be.called + expect(error).to.not.be.called + + this.incomingRes = {} + + this.end() + }) + + incomingResponse.callsFake(function () { + expect(incomingRequest).to.be.calledOnce + expect(error).to.not.be.called + + this.end() + }) + + return new Http({ config, getRemoteState, middleware }) + .handle({}, {}) + .then(function () { + expect(incomingRequest).to.be.calledOnce + expect(incomingResponse).to.be.calledOnce + expect(error).to.not.be.called + }) + }) + + it('moves to Error stack if err in IncomingRequest', function () { + incomingRequest.throws(new Error('oops')) + + error.callsFake(function () { + expect(this.error.message).to.eq('oops') + this.end() + }) + + return new Http({ config, getRemoteState, middleware }) + .handle({}, {}) + .then(function () { + expect(incomingRequest).to.be.calledOnce + expect(incomingResponse).to.not.be.called + expect(error).to.be.calledOnce + }) + }) + + it('moves to Error stack if err in IncomingResponse', function () { + incomingRequest.callsFake(function () { + this.incomingRes = {} + this.end() + }) + + incomingResponse.throws(new Error('oops')) + + error.callsFake(function () { + expect(this.error.message).to.eq('oops') + this.end() + }) + + return new Http({ config, getRemoteState, middleware }) + .handle({}, {}) + .then(function () { + expect(incomingRequest).to.be.calledOnce + expect(incomingResponse).to.be.calledOnce + expect(error).to.be.calledOnce + }) + }) + + it('self can be modified by middleware and passed on', function () { + const reqAdded = {} + const resAdded = {} + const errorAdded = {} + + let expectedKeys = ['req', 'res', 'config', 'getRemoteState', 'middleware'] + + incomingRequest.callsFake(function () { + expect(this).to.include.keys(expectedKeys) + this.reqAdded = reqAdded + expectedKeys.push('reqAdded') + this.next() + }) + + const incomingRequest2 = sinon.stub().callsFake(function () { + expect(this).to.include.keys(expectedKeys) + expect(this.reqAdded).to.equal(reqAdded) + this.incomingRes = {} + expectedKeys.push('incomingRes') + this.end() + }) + + incomingResponse.callsFake(function () { + expect(this).to.include.keys(expectedKeys) + this.resAdded = resAdded + expectedKeys.push('resAdded') + this.next() + }) + + const incomingResponse2 = sinon.stub().callsFake(function () { + expect(this).to.include.keys(expectedKeys) + expect(this.resAdded).to.equal(resAdded) + expectedKeys.push('error') + throw new Error('goto error stack') + }) + + error.callsFake(function () { + expect(this.error.message).to.eq('goto error stack') + expect(this).to.include.keys(expectedKeys) + this.errorAdded = errorAdded + this.next() + }) + + const error2 = sinon.stub().callsFake(function () { + expect(this).to.include.keys(expectedKeys) + expect(this.errorAdded).to.equal(errorAdded) + this.end() + }) + + middleware[HttpStages.IncomingRequest].push(incomingRequest2) + middleware[HttpStages.IncomingResponse].push(incomingResponse2) + middleware[HttpStages.Error].push(error2) + + return new Http({ config, getRemoteState, middleware }) + .handle({}, {}) + .then(function () { + [ + incomingRequest, incomingRequest2, + incomingResponse, incomingResponse2, + error, error2, + ].forEach(function (fn) { + expect(fn).to.be.calledOnce + }) + }) + }) + }) +}) diff --git a/packages/proxy/test/unit/http/request-middleware.spec.ts b/packages/proxy/test/unit/http/request-middleware.spec.ts new file mode 100644 index 000000000000..aa2224fb6f42 --- /dev/null +++ b/packages/proxy/test/unit/http/request-middleware.spec.ts @@ -0,0 +1,17 @@ +import _ from 'lodash' +import RequestMiddleware from '../../../lib/http/request-middleware' +import { expect } from 'chai' + +describe('http/request-middleware', function () { + it('exports the members in the correct order', function () { + expect(_.keys(RequestMiddleware)).to.have.ordered.members([ + 'RedirectToClientRouteIfUnloaded', + 'RedirectToClientRouteIfNotProxied', + 'EndRequestsToBlacklistedHosts', + 'MaybeEndRequestWithBufferedResponse', + 'StripUnsupportedAcceptEncoding', + 'MaybeSetBasicAuthHeaders', + 'SendRequestOutgoing', + ]) + }) +}) diff --git a/packages/proxy/test/unit/http/response-middleware.spec.ts b/packages/proxy/test/unit/http/response-middleware.spec.ts new file mode 100644 index 000000000000..8c081d077f91 --- /dev/null +++ b/packages/proxy/test/unit/http/response-middleware.spec.ts @@ -0,0 +1,25 @@ +import _ from 'lodash' +import ResponseMiddleware from '../../../lib/http/response-middleware' +import { expect } from 'chai' + +describe('http/response-middleware', function () { + it('exports the members in the correct order', function () { + expect(_.keys(ResponseMiddleware)).to.have.ordered.members([ + 'Log', + 'PatchExpressSetHeader', + 'SetInjectionLevel', + 'OmitProblematicHeaders', + 'MaybePreventCaching', + 'CopyCookiesFromIncomingRes', + 'MaybeSendRedirectToClient', + 'CopyResponseStatusCode', + 'ClearCyInitialCookie', + 'MaybeEndWithEmptyBody', + 'MaybeGunzipBody', + 'MaybeInjectHtml', + 'MaybeRemoveSecurity', + 'GzipBody', + 'SendResponseBodyToClient', + ]) + }) +}) diff --git a/packages/server/test/unit/buffers_spec.coffee b/packages/proxy/test/unit/http/util/buffers.spec.coffee similarity index 50% rename from packages/server/test/unit/buffers_spec.coffee rename to packages/proxy/test/unit/http/util/buffers.spec.coffee index dad7b1c44c4c..e92335709831 100644 --- a/packages/server/test/unit/buffers_spec.coffee +++ b/packages/proxy/test/unit/http/util/buffers.spec.coffee @@ -1,30 +1,28 @@ -require("../spec_helper") +{ expect } = require("chai") +{ HttpBuffers } = require("../../../../lib/http/util/buffers") -buffers = require("#{root}lib/util/buffers") +describe "http/util/buffers", -> + buffers = null -describe "lib/util/buffers", -> beforeEach -> - buffers.reset() - - afterEach -> - buffers.reset() + @buffers = new HttpBuffers context "#get", -> it "returns buffer by url", -> obj = {url: "foo"} - buffers.set(obj) + @buffers.set(obj) - buffer = buffers.get("foo") + buffer = @buffers.get("foo") expect(buffer).to.deep.eq(obj) it "falls back to setting the port when buffer could not be found", -> obj = {url: "https://www.google.com/"} - buffers.set(obj) + @buffers.set(obj) - buffer = buffers.get("https://www.google.com:443/") + buffer = @buffers.get("https://www.google.com:443/") expect(buffer).to.deep.eq(obj) @@ -32,9 +30,9 @@ describe "lib/util/buffers", -> it "returns buffer by originalUrl", -> obj = {originalUrl: "foo"} - buffers.set(obj) + @buffers.set(obj) - buffer = buffers.getByOriginalUrl("foo") + buffer = @buffers.getByOriginalUrl("foo") expect(buffer).to.deep.eq(obj) @@ -42,25 +40,25 @@ describe "lib/util/buffers", -> it "removes the found buffer", -> obj = {url: "https://www.google.com/"} - buffers.set(obj) + @buffers.set(obj) - expect(buffers.all()).to.have.length(1) + expect(@buffers.all()).to.have.length(1) - buffer = buffers.take("https://www.google.com:443/") + buffer = @buffers.take("https://www.google.com:443/") expect(buffer).to.deep.eq(obj) - expect(buffers.all()).to.have.length(0) + expect(@buffers.all()).to.have.length(0) it "does not remove anything when not found", -> obj = {url: "https://www.google.com/"} - buffers.set(obj) + @buffers.set(obj) - expect(buffers.all()).to.have.length(1) + expect(@buffers.all()).to.have.length(1) - buffer = buffers.take("asdf") + buffer = @buffers.take("asdf") expect(buffer).to.be.undefined - expect(buffers.all()).to.have.length(1) + expect(@buffers.all()).to.have.length(1) diff --git a/packages/server/test/unit/security_spec.coffee b/packages/proxy/test/unit/http/util/security.spec.coffee similarity index 94% rename from packages/server/test/unit/security_spec.coffee rename to packages/proxy/test/unit/http/util/security.spec.coffee index 9eabf0b10dc6..601e2d046b55 100644 --- a/packages/server/test/unit/security_spec.coffee +++ b/packages/proxy/test/unit/http/util/security.spec.coffee @@ -1,11 +1,10 @@ -require("../spec_helper") - _ = require("lodash") +fs = require("fs") rp = require("request-promise") concat = require("concat-stream") -fs = require("#{root}lib/util/fs") -security = require("#{root}lib/util/security") -Fixtures = require("#{root}test/support/helpers/fixtures") +{ expect } = require("chai") +Promise = require("bluebird") +security = require("../../../../lib/http/util/security") original = """ @@ -137,7 +136,7 @@ expected = """ """ -describe "lib/util/security", -> +describe "http/util/security", -> context ".strip", -> it "replaces obstructive code", -> expect(security.strip(original)).to.eq(expected) @@ -220,20 +219,19 @@ describe "lib/util/security", -> _.each libs, (url, lib) -> it "does not alter code from: '#{lib}'", -> - nock.enableNetConnect() - @timeout(10000) - pathToLib = Fixtures.path("server/libs/#{lib}") + pathToLib = "/tmp/#{lib}" downloadFile = -> rp(url) .then (resp) -> - fs - .outputFileAsync(pathToLib, resp) + Promise.fromCallback (cb) => + fs.writeFile(pathToLib, resp, cb) .return(resp) - fs - .readFileAsync(pathToLib, "utf8") + + Promise.fromCallback (cb) => + fs.readFile(pathToLib, "utf8", cb) .catch(downloadFile) .then (libCode) -> stripped = security.strip(libCode) @@ -253,7 +251,6 @@ describe "lib/util/security", -> try expect(stripped).to.eq(libCode) catch err - fs.outputFileSync(pathToLib + "Diff", stripped) throw new Error("code from '#{lib}' was different") context ".stripStream", -> diff --git a/packages/proxy/tsconfig.json b/packages/proxy/tsconfig.json new file mode 100644 index 000000000000..6e4f8367c920 --- /dev/null +++ b/packages/proxy/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "./../ts/tsconfig.json", + "include": [ + "*.ts", + "lib/*.ts", + "lib/**/*.ts" + ], + "files": [ + "./../ts/index.d.ts" + ] +} diff --git a/packages/server/lib/controllers/proxy.coffee b/packages/server/lib/controllers/proxy.coffee deleted file mode 100644 index e7e5e86c23e8..000000000000 --- a/packages/server/lib/controllers/proxy.coffee +++ /dev/null @@ -1,408 +0,0 @@ -_ = require("lodash") -zlib = require("zlib") -charset = require("charset") -concat = require("concat-stream") -iconv = require("iconv-lite") -Promise = require("bluebird") -accept = require("http-accept") -debug = require("debug")("cypress:server:proxy") -cwd = require("../cwd") -cors = require("../util/cors") -buffers = require("../util/buffers") -rewriter = require("../util/rewriter") -blacklist = require("../util/blacklist") -conditional = require("../util/conditional_stream") -{ passthruStream } = require("../util/passthru_stream") - -REDIRECT_STATUS_CODES = [301, 302, 303, 307, 308] -NO_BODY_STATUS_CODES = [204, 304] - -zlib = Promise.promisifyAll(zlib) - -zlibOptions = { - flush: zlib.Z_SYNC_FLUSH - finishFlush: zlib.Z_SYNC_FLUSH -} - -## https://github.com/cypress-io/cypress/issues/1543 -getNodeCharsetFromResponse = (headers, body) -> - httpCharset = (charset(headers, body, 1024) || '').toLowerCase() - - debug("inferred charset from response %o", { httpCharset }) - - if iconv.encodingExists(httpCharset) - return httpCharset - - ## browsers default to latin1 - return "latin1" - -isGzipError = (err) -> - Object.prototype.hasOwnProperty.call(zlib.constants, err.code) - -## https://github.com/cypress-io/cypress/issues/4298 -## https://tools.ietf.org/html/rfc7230#section-3.3.3 -## HEAD, 1xx, 204, and 304 responses should never contain anything after headers -responseMustHaveEmptyBody = (method, statusCode) -> - _.some([ - _.includes(NO_BODY_STATUS_CODES, statusCode), - _.inRange(statusCode, 100, 200), - _.invoke(method, 'toLowerCase') == 'head', - ]) - -setCookie = (res, key, val, domainName) -> - ## cannot use res.clearCookie because domain - ## is not sent correctly - options = { - domain: domainName - } - - if not val - val = "" - - ## force expires to be the epoch - options.expires = new Date(0) - - res.cookie(key, val, options) - -reqNeedsBasicAuthHeaders = (req, remoteState) -> - { auth, origin } = remoteState - - auth && - not req.headers["authorization"] && - cors.urlMatchesOriginProtectionSpace(req.proxiedUrl, origin) - -module.exports = { - handle: (req, res, config, getRemoteState, request, nodeProxy) -> - remoteState = getRemoteState() - - debug("handling proxied request %o", { - url: req.url - proxiedUrl: req.proxiedUrl - headers: req.headers - remoteState - }) - - ## if we have an unload header it means - ## our parent app has been navigated away - ## directly and we need to automatically redirect - ## to the clientRoute - if req.cookies["__cypress.unload"] - return res.redirect config.clientRoute - - ## when you access cypress from a browser which has not - ## had its proxy setup then req.url will match req.proxiedUrl - ## and we'll know to instantly redirect them to the correct - ## client route - if req.url is req.proxiedUrl and not remoteState.visiting - ## if we dont have a remoteState.origin that means we're initially - ## requesting the cypress app and we need to redirect to the - ## root path that serves the app - return res.redirect(config.clientRoute) - - ## if we have black listed hosts - if blh = config.blacklistHosts - ## and url matches any of our blacklisted hosts - if matched = blacklist.matches(req.proxiedUrl, blh) - ## then bail and return with 503 - ## and set a custom header - res.set("x-cypress-matched-blacklisted-host", matched) - - debug("blacklisting request %o", { - url: req.proxiedUrl - matched: matched - }) - - return res.status(503).end() - - thr = passthruStream() - - @getHttpContent(thr, req, res, remoteState, config, request) - .pipe(res) - - getHttpContent: (thr, req, res, remoteState, config, request) -> - process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" - - isInitial = req.cookies["__cypress.initial"] is "true" - - wantsInjection = null - wantsSecurityRemoved = null - - resContentTypeIs = (respHeaders, str) -> - contentType = respHeaders["content-type"] - - ## make sure the response includes string type - contentType and contentType.includes(str) - - resContentTypeIsJavaScript = (respHeaders) -> - _.some [ - 'application/javascript', - 'application/x-javascript', - 'text/javascript' - ].map(_.partial(resContentTypeIs, respHeaders)) - - reqAcceptsHtml = -> - ## don't inject if this is an XHR from jquery - return if req.headers["x-requested-with"] - - types = accept.parser(req.headers.accept) ? [] - - find = (type) -> - type in types - - ## bail if we didn't find both text/html and application/xhtml+xml - ## https://github.com/cypress-io/cypress/issues/288 - find("text/html") and find("application/xhtml+xml") - - resMatchesOriginPolicy = (respHeaders) -> - switch remoteState.strategy - when "http" - cors.urlMatchesOriginPolicyProps(req.proxiedUrl, remoteState.props) - when "file" - req.proxiedUrl.startsWith(remoteState.origin) - - setCookies = (value) -> - ## dont modify any cookies if we're trying to clear - ## the initial cookie and we're not injecting anything - return if (not value) and (not wantsInjection) - - ## dont set the cookies if we're not on the initial request - return if not isInitial - - setCookie(res, "__cypress.initial", value, remoteState.domainName) - - setBody = (str, statusCode, headers) -> - ## set the status to whatever the incomingRes statusCode is - res.status(statusCode) - - ## turn off __cypress.initial by setting false here - setCookies(false, wantsInjection) - - encoding = headers["content-encoding"] - - isGzipped = encoding and encoding.includes("gzip") - - debug("received response for %o", { - url: req.proxiedUrl - headers, - statusCode, - isGzipped - wantsInjection, - wantsSecurityRemoved, - }) - - if responseMustHaveEmptyBody(req.method, statusCode) - return res.end() - - ## if there is nothing to inject then just - ## bypass the stream buffer and pipe this back - if wantsInjection - rewrite = (body) -> - ## transparently decode their body to a node string and then re-encode - nodeCharset = getNodeCharsetFromResponse(headers, body) - body = rewriter.html(iconv.decode(body, nodeCharset), remoteState.domainName, wantsInjection, wantsSecurityRemoved) - iconv.encode(body, nodeCharset) - - ## TODO: we can probably move this to the new - ## replacestream rewriter instead of using - ## a buffer - injection = concat (body) -> - ## if we're gzipped that means we need to unzip - ## this content first, inject, and the rezip - if isGzipped - zlib.gunzipAsync(body, zlibOptions) - .then(rewrite) - .then(zlib.gzipAsync) - .then(thr.end) - ## if we have an error here there's nothing - ## to do but log it out and end the socket - ## because we cannot inject into content - ## that failed rewriting gzip - ## which is the same thing we do below - ## on regular proxied network requests - .catch(endWithNetworkErr) - else - thr.end rewrite(body) - - str.pipe(injection) - else - ## only rewrite if we should - if wantsSecurityRemoved - gunzip = zlib.createGunzip(zlibOptions) - gunzip.setEncoding("utf8") - - onError = (err) -> - gzipError = isGzipError(err) - - debug("failed to proxy response %o", { - url: req.proxiedUrl - headers - statusCode - isGzipped - gzipError - wantsInjection - wantsSecurityRemoved - err - }) - - endWithNetworkErr(err) - - ## only unzip when it is already gzipped - return str - .pipe(conditional(isGzipped, gunzip)) - .on("error", onError) - .pipe(rewriter.security()) - .on("error", onError) - .pipe(conditional(isGzipped, zlib.createGzip())) - .on("error", onError) - .pipe(thr) - .on("error", onError) - - return str.pipe(thr) - - endWithNetworkErr = (err) -> - debug('request failed in proxy layer %o', { - res: _.pick(res, 'headersSent', 'statusCode', 'headers') - req: _.pick(req, 'url', 'proxiedUrl', 'headers', 'method') - err - }) - - req.socket.destroy() - - onResponse = (str, incomingRes) => - {headers, statusCode} = incomingRes - - originalSetHeader = res.setHeader - - ## express does all kinds of silly/nasty stuff to the content-type... - ## but we don't want to change it at all! - res.setHeader = (k, v) -> - if k == 'content-type' - v = incomingRes.headers['content-type'] - - originalSetHeader.call(res, k, v) - - wantsInjection ?= do -> - return false if not resContentTypeIs(headers, "text/html") - - return false if not resMatchesOriginPolicy(headers) - - return "full" if isInitial - - return false if not reqAcceptsHtml() - - return "partial" - - wantsSecurityRemoved = do -> - ## we want to remove security IF we're doing a full injection - ## on the response or its a request for any javascript script tag - config.modifyObstructiveCode and ( - (wantsInjection is "full") or - resContentTypeIsJavaScript(headers) - ) - - @setResHeaders(req, res, incomingRes, wantsInjection) - - ## always proxy the cookies coming from the incomingRes - if cookies = headers["set-cookie"] - ## normalize into array - for c in [].concat(cookies) - try - res.append("Set-Cookie", c) - catch err - ## noop - - if REDIRECT_STATUS_CODES.includes(statusCode) - newUrl = headers.location - - ## set cookies to initial=true - setCookies(true) - - debug("redirecting to new url %o", { status: statusCode, url: newUrl }) - - ## finally redirect our user agent back to our domain - ## by making this an absolute-path-relative redirect - return res.redirect(statusCode, newUrl) - - if headers["x-cypress-file-server-error"] - wantsInjection or= "partial" - - setBody(str, statusCode, headers) - - if obj = buffers.take(req.proxiedUrl) - wantsInjection = "full" - - onResponse(obj.stream, obj.response) - else - opts = { - timeout: null - strictSSL: false - followRedirect: false - retryIntervals: [0, 100, 200, 200] - } - - ## strip unsupported accept-encoding headers - encodings = accept.parser(req.headers["accept-encoding"]) ? [] - - if "gzip" in encodings - ## we only want to support gzip right now - req.headers["accept-encoding"] = "gzip" - else - ## else just delete them since we cannot - ## properly decode them - delete req.headers["accept-encoding"] - - if remoteState.strategy is "file" and req.proxiedUrl.startsWith(remoteState.origin) - opts.url = req.proxiedUrl.replace(remoteState.origin, remoteState.fileServer) - else - opts.url = req.proxiedUrl - - ## if we have auth headers and this request matches our origin - ## protection space and the user has not supplied auth headers - if reqNeedsBasicAuthHeaders(req, remoteState) - { auth } = remoteState - - base64 = Buffer - .from(auth.username + ":" + auth.password) - .toString("base64") - - req.headers["authorization"] = "Basic #{base64}" - - rq = request.create(opts) - - rq.on("error", endWithNetworkErr) - - rq.on "response", (incomingRes) -> - onResponse(rq, incomingRes) - - ## if our original request has been - ## aborted, then ensure we forward - ## this onto the proxied request - ## https://github.com/cypress-io/cypress/issues/2612 - ## this can happen on permanent connections - ## like SSE, but also on any regular ol' - ## http request - req.on "aborted", -> - rq.abort() - - ## proxy the request body, content-type, headers - ## to the new rq - req.pipe(rq) - - return thr - - setResHeaders: (req, res, incomingRes, wantsInjection) -> - return if res.headersSent - - ## omit problematic headers - headers = _.omit incomingRes.headers, "set-cookie", "x-frame-options", "content-length", "content-security-policy", "connection" - - ## do not cache when we inject content into responses - ## later on we should switch to an etag system so we dont - ## have to download the remote http responses if the etag - ## hasnt changed - if wantsInjection - headers["cache-control"] = "no-cache, no-store, must-revalidate" - - ## proxy the headers - res.set(headers) -} diff --git a/packages/server/lib/routes.coffee b/packages/server/lib/routes.coffee index 449fc75ce41b..03ccefe8540a 100644 --- a/packages/server/lib/routes.coffee +++ b/packages/server/lib/routes.coffee @@ -12,10 +12,9 @@ runner = require("./controllers/runner") xhrs = require("./controllers/xhrs") client = require("./controllers/client") files = require("./controllers/files") -proxy = require("./controllers/proxy") staticCtrl = require("./controllers/static") -module.exports = (app, config, request, getRemoteState, project, nodeProxy) -> +module.exports = (app, config, request, getRemoteState, project, networkProxy) -> ## routing for the actual specs which are processed automatically ## this could be just a regular .js file or a .coffee file app.get "/__cypress/tests", (req, res, next) -> @@ -71,7 +70,7 @@ module.exports = (app, config, request, getRemoteState, project, nodeProxy) -> }) app.all "*", (req, res, next) -> - proxy.handle(req, res, config, getRemoteState, request, nodeProxy) + networkProxy.handleHttpRequest(req, res) ## when we experience uncaught errors ## during routing just log them out to diff --git a/packages/server/lib/server.coffee b/packages/server/lib/server.coffee index a06491932dfe..cb3dd2448a34 100644 --- a/packages/server/lib/server.coffee +++ b/packages/server/lib/server.coffee @@ -14,14 +14,11 @@ check = require("check-more-types") httpsProxy = require("@packages/https-proxy") compression = require("compression") debug = require("debug")("cypress:server:server") -agent = require("@packages/network").agent -cors = require("./util/cors") -uri = require("./util/uri") +{ agent, blacklist, cors, uri } = require("@packages/network") +{ NetworkProxy } = require("@packages/proxy") origin = require("./util/origin") ensureUrl = require("./util/ensure-url") appData = require("./util/app_data") -buffers = require("./util/buffers") -blacklist = require("./util/blacklist") statusCode = require("./util/status_code") headersUtil = require("./util/headers") allowDestroy = require("./util/server_destroy") @@ -132,25 +129,23 @@ class Server la(_.isPlainObject(config), "expected plain config object", config) Promise.try => - ## always reset any buffers - ## TODO: change buffers to be an instance - ## here and pass this dependency around - buffers.reset() - app = @createExpressApp(config.morgan) logger.setSettings(config) ## generate our request instance ## and set the responseTimeout + ## TODO: might not be needed anymore @_request = Request({timeout: config.responseTimeout}) @_nodeProxy = httpProxy.createProxyServer() getRemoteState = => @_getRemoteState() + @_networkProxy = new NetworkProxy({ config, getRemoteState }) + @createHosts(config.hosts) - @createRoutes(app, config, @_request, getRemoteState, project, @_nodeProxy) + @createRoutes(app, config, @_request, getRemoteState, project, @_networkProxy) @createServer(app, config, project, @_request, onWarning) @@ -366,7 +361,7 @@ class Server ## then just respond with its details ## so we are idempotant and do not make ## another request - if obj = buffers.getByOriginalUrl(urlStr) + if obj = @_networkProxy.getHttpBuffer(urlStr) debug("got previous request buffer for url:", urlStr) ## reset the cookies from the existing stream's jar @@ -482,7 +477,7 @@ class Server responseBufferStream.end(responseBuffer) - buffers.set({ + @_networkProxy.setHttpBuffer({ url: newUrl jar: jar stream: responseBufferStream @@ -675,7 +670,7 @@ class Server socket.end() if socket.writable reset: -> - buffers.reset() + @_networkProxy?.reset() @_onDomainSet(@_baseUrl ? "") diff --git a/packages/server/lib/util/blacklist.js b/packages/server/lib/util/blacklist.js deleted file mode 100644 index 88ade6bec29c..000000000000 --- a/packages/server/lib/util/blacklist.js +++ /dev/null @@ -1,22 +0,0 @@ -const _ = require('lodash') -const minimatch = require('minimatch') -const uri = require('./uri') - -const matches = function (urlToCheck, blacklistHosts) { - //# normalize into flat array - blacklistHosts = [].concat(blacklistHosts) - - urlToCheck = uri.stripProtocolAndDefaultPorts(urlToCheck) - - // use minimatch against the url - // to see if any match - const matchUrl = (hostMatcher) => { - return minimatch(urlToCheck, hostMatcher) - } - - return _.find(blacklistHosts, matchUrl) -} - -module.exports = { - matches, -} diff --git a/packages/server/lib/util/buffers.js b/packages/server/lib/util/buffers.js deleted file mode 100644 index 3bec3c8e23e2..000000000000 --- a/packages/server/lib/util/buffers.js +++ /dev/null @@ -1,66 +0,0 @@ -const _ = require('lodash') -const debug = require('debug')('cypress:server:buffers') -const uri = require('./uri') - -let buffers = [] - -module.exports = { - all () { - return buffers - }, - - keys () { - return _.map(buffers, 'url') - }, - - reset () { - debug('resetting buffers') - - buffers = [] - }, - - set (obj = {}) { - return buffers.push(_.pick(obj, 'url', 'originalUrl', 'jar', 'stream', 'response', 'details')) - }, - - getByOriginalUrl (str) { - return _.find(buffers, { originalUrl: str }) - }, - - get (str) { - const find = (str) => { - return _.find(buffers, { url: str }) - } - - const b = find(str) - - if (b) { - return b - } - - let parsed = uri.parse(str) - - //# if we're on https and we have a port - //# then attempt to find the buffer by - //# slicing off the port since our buffer - //# was likely stored without a port - if ((parsed.protocol === 'https:') && parsed.port) { - parsed = uri.removePort(parsed) - - return find(parsed.format()) - } - }, - - take (str) { - const buffer = this.get(str) - - if (buffer) { - buffers = _.without(buffers, buffer) - - debug('found request buffer %o', { buffer: _.pick(buffer, 'url', 'originalUrl'), bufferCount: buffers.length }) - } - - return buffer - }, - -} diff --git a/packages/server/lib/util/conditional_stream.js b/packages/server/lib/util/conditional_stream.js deleted file mode 100644 index 4d28343cf6bc..000000000000 --- a/packages/server/lib/util/conditional_stream.js +++ /dev/null @@ -1,11 +0,0 @@ -const stream = require('stream') - -module.exports = (condition, dest) => { - // if truthy return the dest stream - if (condition) { - return dest - } - - // else passthrough the stream - return stream.PassThrough() -} diff --git a/packages/server/lib/util/cors.js b/packages/server/lib/util/cors.js deleted file mode 100644 index e0608f766811..000000000000 --- a/packages/server/lib/util/cors.js +++ /dev/null @@ -1,68 +0,0 @@ -const _ = require('lodash') -const url = require('url') -const uri = require('./uri') -const debug = require('debug')('cypress:server:cors') -const parseDomain = require('parse-domain') - -const ipAddressRe = /^[\d\.]+$/ - -module.exports = { - parseUrlIntoDomainTldPort (str) { - let { hostname, port, protocol } = url.parse(str) - - if (port == null) { - port = protocol === 'https:' ? '443' : '80' - } - - let parsed = parseDomain(hostname, { - privateTlds: true, // use the public suffix - customTlds: ipAddressRe, - }) - - // if we couldn't get a parsed domain - if (!parsed) { - // then just fall back to a dumb check - // based on assumptions that the tld - // is the last segment after the final - // '.' and that the domain is the segment - // before that - const segments = hostname.split('.') - - parsed = { - tld: segments[segments.length - 1] || '', - domain: segments[segments.length - 2] || '', - } - } - - const obj = {} - - obj.port = port - obj.tld = parsed.tld - obj.domain = parsed.domain - // obj.protocol = protocol - - debug('Parsed URL %o', obj) - - return obj - }, - - urlMatchesOriginPolicyProps (urlStr, props) { - // take a shortcut here in the case - // where remoteHostAndPort is null - if (!props) { - return false - } - - const parsedUrl = this.parseUrlIntoDomainTldPort(urlStr) - - // does the parsedUrl match the parsedHost? - return _.isEqual(parsedUrl, props) - }, - - urlMatchesOriginProtectionSpace (urlStr, origin) { - const normalizedUrl = uri.addDefaultPort(urlStr).format() - const normalizedOrigin = uri.addDefaultPort(origin).format() - - return _.startsWith(normalizedUrl, normalizedOrigin) - }, -} diff --git a/packages/server/lib/util/inject.js b/packages/server/lib/util/inject.js deleted file mode 100644 index e2745c2b3534..000000000000 --- a/packages/server/lib/util/inject.js +++ /dev/null @@ -1,27 +0,0 @@ -const { oneLine } = require('common-tags') - -module.exports = { - partial (domain) { - return oneLine` - - ` - }, - - full (domain) { - return oneLine` - - ` - }, -} diff --git a/packages/server/package.json b/packages/server/package.json index a4b162ca006c..8697929bf833 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -50,7 +50,6 @@ "browserify": "16.3.0", "chai": "1.10.0", "chalk": "2.4.2", - "charset": "1.0.1", "check-more-types": "2.24.0", "chokidar": "3.0.1", "cjsxify": "0.3.0", @@ -77,11 +76,9 @@ "getos": "3.1.1", "glob": "7.1.3", "graceful-fs": "4.2.0", - "http-accept": "0.1.6", "http-proxy": "1.17.0", "http-status-codes": "1.3.2", "human-interval": "0.1.6", - "iconv-lite": "0.5.0", "image-size": "0.7.4", "is-fork-pr": "2.3.0", "is-html": "2.0.0", @@ -107,11 +104,9 @@ "opn": "cypress-io/opn#2f4e9a216ca7bdb95dfae9d46d99ddf004b3cbb5", "ospath": "1.2.2", "p-queue": "6.1.0", - "parse-domain": "2.0.0", "pumpify": "1.5.1", "ramda": "0.24.1", "randomstring": "1.1.5", - "replacestream": "4.0.3", "request": "2.88.0", "request-promise": "4.2.4", "return-deep-diff": "0.3.0", diff --git a/packages/server/test/integration/http_requests_spec.coffee b/packages/server/test/integration/http_requests_spec.coffee index cacd50aeb81e..7256b7ff7f32 100644 --- a/packages/server/test/integration/http_requests_spec.coffee +++ b/packages/server/test/integration/http_requests_spec.coffee @@ -2390,8 +2390,7 @@ describe "Routes", -> "Cookie": "__cypress.initial=false" } - if type? - headers["Accept"] = type + headers["Accept"] = type @rp({ url: "http://www.google.com/iframe" diff --git a/packages/server/test/integration/server_spec.coffee b/packages/server/test/integration/server_spec.coffee index 92f96ced8427..d645a0a3ebe6 100644 --- a/packages/server/test/integration/server_spec.coffee +++ b/packages/server/test/integration/server_spec.coffee @@ -6,7 +6,6 @@ rp = require("request-promise") Promise = require("bluebird") evilDns = require("evil-dns") httpsServer = require("#{root}../https-proxy/test/helpers/https_server") -buffers = require("#{root}lib/util/buffers") config = require("#{root}lib/config") Server = require("#{root}lib/server") Fixtures = require("#{root}test/support/helpers/fixtures") @@ -83,6 +82,8 @@ describe "Server", -> @proxy = "http://localhost:" + port + @buffers = @server._networkProxy.http.buffers + @fileServer = @server._fileServer.address() ]) @@ -168,7 +169,7 @@ describe "Server", -> cookies: [] }) - expect(buffers.keys()).to.deep.eq(["http://localhost:2000/index.html"]) + expect(@buffers.keys()).to.deep.eq(["http://localhost:2000/index.html"]) .then => @server._onResolveUrl("/index.html", {}, @automationRequest) .then (obj = {}) => @@ -193,7 +194,7 @@ describe "Server", -> expect(res.body).to.include("document.domain") expect(res.body).to.include("localhost") expect(res.body).to.include("Cypress") - expect(buffers.keys()).to.deep.eq([]) + expect(@buffers.keys()).to.deep.eq([]) it "can follow static file redirects", -> @server._onResolveUrl("/sub", {}, @automationRequest) @@ -264,13 +265,13 @@ describe "Server", -> cookies: [] }) - expect(buffers.keys()).to.deep.eq(["http://localhost:2000/index.html"]) + expect(@buffers.keys()).to.deep.eq(["http://localhost:2000/index.html"]) .then => @rp("http://localhost:2000/index.html") - .then (res) -> + .then (res) => expect(res.statusCode).to.eq(200) - expect(buffers.keys()).to.deep.eq([]) + expect(@buffers.keys()).to.deep.eq([]) describe "http", -> beforeEach -> @@ -525,7 +526,7 @@ describe "Server", -> ] }) - expect(buffers.keys()).to.deep.eq(["http://espn.go.com/"]) + expect(@buffers.keys()).to.deep.eq(["http://espn.go.com/"]) .then => @server._onResolveUrl("http://espn.com/", {}, @automationRequest) .then (obj = {}) => @@ -552,7 +553,7 @@ describe "Server", -> expect(res.body).to.include("document.domain") expect(res.body).to.include("go.com") expect(res.body).to.include("Cypress.action('app:window:before:load', window); espn") - expect(buffers.keys()).to.deep.eq([]) + expect(@buffers.keys()).to.deep.eq([]) it "does not buffer 'bad' responses", -> sinon.spy(@server._request, "sendStream") @@ -651,7 +652,7 @@ describe "Server", -> }) @server._onResolveUrl("http://getbootstrap.com/#/foo", {}, @automationRequest) - .then (obj = {}) -> + .then (obj = {}) => expectToEqDetails(obj, { isOkStatusCode: true isHtml: true @@ -664,13 +665,13 @@ describe "Server", -> cookies: [] }) - expect(buffers.keys()).to.deep.eq(["http://getbootstrap.com/"]) + expect(@buffers.keys()).to.deep.eq(["http://getbootstrap.com/"]) .then => @rp("http://getbootstrap.com/") - .then (res) -> + .then (res) => expect(res.statusCode).to.eq(200) - expect(buffers.keys()).to.deep.eq([]) + expect(@buffers.keys()).to.deep.eq([]) it "can serve non 2xx status code requests when option set", -> nock("http://google.com") diff --git a/packages/server/test/unit/proxy_spec.coffee b/packages/server/test/unit/proxy_spec.coffee deleted file mode 100644 index 9cd85af1e6f2..000000000000 --- a/packages/server/test/unit/proxy_spec.coffee +++ /dev/null @@ -1,133 +0,0 @@ -## THESE TESTS ARE COMMENTED OUT BECAUSE ALL OF REMOTE_INITIAL -## WAS REFACTORED AND MOST DO NOT APPLY. -## WILL ADD UNIT TESTS AS PROBLEMS ARISE (IF THEY ARISE) - -require("../spec_helper") - -Readable = require("stream").Readable -proxy = require("#{root}lib/controllers/proxy") - -describe "lib/proxy", -> - -# it "injects content", (done) -> -# readable = new Readable - -# readable.push('') -# readable.push(null) - -# readable.pipe(proxy.injectContent("wow")) -# .pipe through (d) -> -# expect(d.toString()).to.eq(" wow") -# done() - -# describe "redirects", -> -# beforeEach -> -# @req = { -# url: "/__remote/#{@baseUrl}/", -# session: {} -# } - -# nock(@baseUrl) -# .get("/") -# .reply(301, "", { -# 'location': @redirectUrl -# }) - -# @res.redirect = (loc) => -# @req.url = loc -# proxy.handle(@req, @res) - -# it "redirects on 301", (done) -> -# nock(@redirectUrl) -# .get("/") -# .reply(200, => -# done() -# ) - -# proxy.handle(@req, @res) - -# it "resets session remote after a redirect", (done) -> -# nock(@redirectUrl) -# .get("/") -# .reply(200, => -# expect(@req.session.remote).to.eql("http://x.com") -# done() -# ) - -# proxy.handle(@req, @res) - -# context "#parseReqUrl", -> -# it "removes /__remote/", -> -# url = proxy.parseReqUrl("/__remote/www.github.com") -# expect(url).to.eq "www.github.com" - -# it "removes __initial query param", -> -# url = proxy.parseReqUrl("/__remote/www.github.com?__initial=true") -# expect(url).to.eq "www.github.com" - -# it "leaves other query params", -> -# url = proxy.parseReqUrl("/__remote/www.github.com?__initial=true&foo=bar") -# expect(url).to.eq "www.github.com/?foo=bar" - -# it "doesnt strip trailing slashes", -> -# url = proxy.parseReqUrl("/__remote/www.github.com/") -# expect(url).to.eq "www.github.com/" - -# context "#prepareUrlForRedirect", -> -# it "prepends with /__remote/ and adds __initial=true query param", -> -# url = proxy.prepareUrlForRedirect("www.github.com", "www.github.com/bar") -# expect(url).to.eq "/__remote/www.github.com/bar?__initial=true" - -# it "doesnt strip leading slashes", -> -# url = proxy.prepareUrlForRedirect("www.github.com", "www.github.com/") -# expect(url).to.eq "/__remote/www.github.com/?__initial=true" - -# it "handles url leading slashes", -> -# url = proxy.prepareUrlForRedirect("www.github.com/foo", "www.github.com/foo/") -# expect(url).to.eq "/__remote/www.github.com/foo/?__initial=true" - -# it "handles existing query params", -> -# url = proxy.prepareUrlForRedirect("www.github.com", "www.github.com/foo?bar=baz") -# expect(url).to.eq "/__remote/www.github.com/foo?bar=baz&__initial=true" - -# context "setting session", -> -# beforeEach -> -# nock(@baseUrl) -# .get("/") -# .reply(200) - -# nock(@baseUrl) -# .get("/?foo=bar") -# .reply(200) - -# it "sets immediately before requests", -> -# @req = -# url: "/__remote/#{@baseUrl}" -# session: {} - -# proxy.handle(@req, @res) - -# expect(@req.session.remote).to.eql(@baseUrl) - -# it "does not include query params in the url", -> -# @req = -# url: "/__remote/#{@baseUrl}?foo=bar" -# session: {} - -# proxy.handle(@req, @res) -# expect(@req.session.remote).to.eql(@baseUrl) - -# context "relative files", -> -# it "#getRelativeFileContent strips trailing slashes", -> -# createReadStream = sinon.stub(fs, "createReadStream") -# proxy.getRelativeFileContent("index.html/", {}) -# expect(createReadStream).to.be.calledWith("/Users/brian/app/index.html") - -# context "absolute files", -> - -# context "file files", -> - -# context "errors", -> -# it "bubbles 500's from external server" - -# it "throws on authentication required" diff --git a/packages/server/test/unit/server_spec.coffee b/packages/server/test/unit/server_spec.coffee index 5bad0ce26ece..6edc2731943f 100644 --- a/packages/server/test/unit/server_spec.coffee +++ b/packages/server/test/unit/server_spec.coffee @@ -13,7 +13,6 @@ Server = require("#{root}lib/server") Socket = require("#{root}lib/socket") fileServer = require("#{root}lib/file_server") ensureUrl = require("#{root}lib/util/ensure-url") -buffers = require("#{root}lib/util/buffers") morganFn = -> mockery.registerMock("morgan", -> morganFn) @@ -193,11 +192,14 @@ describe "lib/server", -> context "#reset", -> beforeEach -> - sinon.stub(buffers, "reset") + @server.open(@config) + .then => + @buffers = @server._networkProxy.http + sinon.stub(@buffers, "reset") it "resets the buffers", -> @server.reset() - expect(buffers.reset).to.be.called + expect(@buffers.reset).to.be.called it "sets the domain to the previous base url if set", -> @server._baseUrl = "http://localhost:3000" diff --git a/packages/ts/index.d.ts b/packages/ts/index.d.ts index 65d38a99e2e6..26a677f4db8f 100644 --- a/packages/ts/index.d.ts +++ b/packages/ts/index.d.ts @@ -89,3 +89,9 @@ declare module 'proxy-from-env' { declare interface SymbolConstructor { for(str: string): SymbolConstructor } + +declare module 'url' { + interface UrlWithStringQuery { + format(): string + } +}