diff --git a/README.md b/README.md index 1059fc9..302d5d3 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,10 @@ Saving the `request` module as a dependency is compulsory. npm install --save request ``` +Support for Brotli encoded responses is enabled by default when using Node.js v10 or later. +If you wish to enable support for older Node.js versions, you may install [brotli](https://npmjs.com/package/brotli). +It is recommended but not required. + Usage ============ Cloudscraper uses `request-promise` by default since v3. You can find the migration guide [here.](docs/migration-guide.md) @@ -128,7 +132,11 @@ var options = { // Support only this max challenges in row. If CF returns more, throw an error challengesToSolve: 3, // Remove Cloudflare's email protection, replace encoded email with decoded versions - decodeEmails: false + decodeEmails: false, + // Support gzip encoded responses (Should be enabled unless using custom headers) + gzip: true, + // Removes a few problematic TLSv1.0 ciphers to avoid CAPTCHA + agentOptions: { ciphers } }; cloudscraper(options).then(console.log); diff --git a/index.js b/index.js index c9dec9a..d9ee523 100644 --- a/index.js +++ b/index.js @@ -4,8 +4,8 @@ const requestModule = require('request-promise'); const sandbox = require('./lib/sandbox'); const decodeEmails = require('./lib/email-decode.js'); const getDefaultHeaders = require('./lib/headers'); -const agentOptions = require('./lib/agent-options'); const brotli = require('./lib/brotli'); +const crypto = require('crypto'); const { RequestError, @@ -37,8 +37,10 @@ function defaults (params) { decodeEmails: false, // Support gzip encoded responses gzip: true, - // Adds secure TLSv1.3 ciphers when using older openssl versions - agentOptions + agentOptions: { + // Removes a few problematic TLSv1.0 ciphers to avoid CAPTCHA + ciphers: crypto.constants.defaultCipherList + ':!ECDHE+SHA:!AES128-SHA' + } }; // Object.assign requires at least nodejs v4, request only test/supports v6+ diff --git a/lib/agent-options.js b/lib/agent-options.js deleted file mode 100644 index b06fd36..0000000 --- a/lib/agent-options.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict'; - -const tls = require('tls'); - -const ciphers = getCiphers(); - -if (ciphers !== -1) { - module.exports.ciphers = ciphers; -} - -function getCiphers () { - // SSL_CTX_set_cipher_list will simply ignore any unsupported ciphers - const defaults = [ - 'TLS_AES_128_CCM_8_SHA256', - 'TLS_AES_128_CCM_SHA256', - 'TLS_AES_128_GCM_SHA256', - 'TLS_AES_256_GCM_SHA384', - 'TLS_CHACHA20_POLY1305_SHA256' - ]; - - // We already have these defaults if using openssl v1.1.1 and later - const v = process.versions.openssl.match(/(\d)+\.(\d+)\.(\d+)/); - if (v[1] >= 1 && v[2] >= 1 && v[3] >= 1) { - return -1; - } - - const suites = tls.getCiphers() - .map(function (s) { - return s.toUpperCase(); - }); - - let missing = false; - // Add the default TLSv1.3 cipher suites if missing - for (let i = 0; i < defaults.length; i++) { - if (suites.indexOf(defaults[i]) === -1) { - missing = true; - suites.push(defaults[i]); - } - } - - return missing ? suites.join(':') : -1; -} diff --git a/test/helper.js b/test/helper.js index 218d104..4d3c9c5 100644 --- a/test/helper.js +++ b/test/helper.js @@ -5,9 +5,10 @@ var url = require('url'); var path = require('path'); var express = require('express'); -// Clone the default headers for tests -var defaultHeaders = Object.assign({}, require('../').defaultParams.headers); -var agentOptions = require('../lib/agent-options'); +// Clone a few defaults before testing +var opts = require('../').defaultParams; +var defaultHeaders = Object.assign({}, opts.headers); +var agentOptions = Object.assign({}, opts.agentOptions); // Cache fixtures so they're only read from fs but once var cache = {}; @@ -33,7 +34,7 @@ var helper = { challengesToSolve: 3, decodeEmails: false, gzip: true, - agentOptions + agentOptions: Object.assign({}, agentOptions) }; }, getFixture: function (fileName) {