From 1d39b350ea0f6c97951e08364995632843fa5bdd Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 22 Mar 2019 02:25:43 +0100 Subject: [PATCH 1/8] Fix for challenge where part of formula is stored in a div --- cfscrape/__init__.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index eed686a..aa07233 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -120,7 +120,7 @@ def solve_challenge(self, body, domain): except Exception: raise ValueError("Unable to identify Cloudflare IUAM Javascript on website. %s" % BUG_REPORT) - js = re.sub(r"a\.value = (.+ \+ t\.length(\).toFixed\(10\))?).+", r"\1", js) + js = re.sub(r"a\.value = (.+\.toFixed\(10\);).+", r"\1", js) js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js).replace("t.length", str(len(domain))) # Strip characters that could be used to exit the string context @@ -130,9 +130,30 @@ def solve_challenge(self, body, domain): if "toFixed" not in js: raise ValueError("Error parsing Cloudflare IUAM Javascript challenge. %s" % BUG_REPORT) + # 2019-03-20: Cloudflare sometimes stores part of the challenge in a div which is later + # added using document.getElementById(x).innerHTML, so it is necessary to simulate that + # method and value. + try: + # Find the id of the div in the javascript code. + k = re.search(r"k\s+=\s+'([^']+)';", body).group(1) + # Find the div with that id and store its content. + val = re.search(r'(.*)' % (k), body).group(3) + except Exception: + # If not available, either the code has been modified again, or the old + # style challenge is used. + k = '' + val = '' + # Use vm.runInNewContext to safely evaluate code # The sandboxed code cannot use the Node.js standard library - js = "console.log(require('vm').runInNewContext('%s', Object.create(null), {timeout: 5000}));" % js + # Add the atob method which is now used by Cloudflares code, but is not available in all node versions. + simulate_document_js = 'var document= {getElementById: function(x) { return {innerHTML:"%s"};}}' % (val) + atob_js = 'var atob = function(str) {return Buffer.from(str, "base64").toString("binary");}' + # t is not defined, so we have to define it and set it to the domain name. + js = '%s;%s;var t="%s";%s' % (simulate_document_js,atob_js,domain,js) + buffer_js = 'var Buffer = require('buffer').Buffer' + # Pass Buffer into the new context, so it is available for atob. + js = "%s;console.log(require('vm').runInNewContext('%s', {'Buffer':Buffer}, {timeout: 5000}));" % (buffer_js, js) try: result = subprocess.check_output(["node", "-e", js]).strip() From fe79caa299e92dd5afd3a8daa9cb8f53988d2420 Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 22 Mar 2019 09:23:04 +0100 Subject: [PATCH 2/8] Fix syntax error. --- cfscrape/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index aa07233..fe6f997 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -151,7 +151,7 @@ def solve_challenge(self, body, domain): atob_js = 'var atob = function(str) {return Buffer.from(str, "base64").toString("binary");}' # t is not defined, so we have to define it and set it to the domain name. js = '%s;%s;var t="%s";%s' % (simulate_document_js,atob_js,domain,js) - buffer_js = 'var Buffer = require('buffer').Buffer' + buffer_js = "var Buffer = require('buffer').Buffer" # Pass Buffer into the new context, so it is available for atob. js = "%s;console.log(require('vm').runInNewContext('%s', {'Buffer':Buffer}, {timeout: 5000}));" % (buffer_js, js) From 2d22225ed5a9f3f3360e430a4ffc20031c010795 Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 22 Mar 2019 22:21:04 +0100 Subject: [PATCH 3/8] Consider query string when building redirect URL --- cfscrape/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index fe6f997..1bc7ae1 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -9,8 +9,10 @@ try: from urlparse import urlparse + from urlparse import urlunparse except ImportError: from urllib.parse import urlparse + from urllib.parse import urlunparse __version__ = "1.9.7" @@ -109,7 +111,7 @@ def solve_cf_challenge(self, resp, **original_kwargs): redirect_location = urlparse(redirect.headers["Location"]) if not redirect_location.netloc: - redirect_url = "%s://%s%s" % (parsed_url.scheme, domain, redirect_location.path) + redirect_url = urlunparse((parsed_url.scheme, domain, redirect_location.path, redirect_location.params, redirect_location.query, redirect_location.fragment)) return self.request(method, redirect_url, **original_kwargs) return self.request(method, redirect.headers["Location"], **original_kwargs) From f4e9f9a2b04150555fb35426be54a12f72b0e66b Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 5 Apr 2019 14:06:31 +0200 Subject: [PATCH 4/8] Change user agent on each instantiation. --- cfscrape/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index 1bc7ae1..9e580ae 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -7,6 +7,8 @@ from requests.sessions import Session +from collections import OrderedDict + try: from urlparse import urlparse from urlparse import urlunparse @@ -26,8 +28,6 @@ "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0" ] -DEFAULT_USER_AGENT = random.choice(DEFAULT_USER_AGENTS) - BUG_REPORT = """\ Cloudflare may have changed their technique, or there may be a bug in the script. @@ -52,7 +52,7 @@ def __init__(self, *args, **kwargs): if "requests" in self.headers["User-Agent"]: # Set a random User-Agent if no custom User-Agent has been set - self.headers["User-Agent"] = DEFAULT_USER_AGENT + self.headers["User-Agent"] = random.choice(DEFAULT_USER_AGENTS) def is_cloudflare_challenge(self, resp): return ( From 814519ce0857542223fe4746b69e26fab96640ce Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 5 Apr 2019 14:07:21 +0200 Subject: [PATCH 5/8] Use cloudflares delay by default before submitting the challenge answer. --- cfscrape/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index 9e580ae..59dfb42 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -47,7 +47,8 @@ class CloudflareScraper(Session): def __init__(self, *args, **kwargs): - self.delay = kwargs.pop("delay", 8) + self.default_delay = 8 + self.delay = kwargs.pop("delay", self.default_delay) super(CloudflareScraper, self).__init__(*args, **kwargs) if "requests" in self.headers["User-Agent"]: @@ -98,6 +99,14 @@ def solve_cf_challenge(self, resp, **original_kwargs): # Solve the Javascript challenge params["jschl_answer"] = self.solve_challenge(body, domain) + # Check if the default delay has been overridden. If not, use the delay required by + # cloudflare. + if self.delay == self.default_delay: + try: + self.delay = float(re.search(r"submit\(\);\r?\n\s*},\s*([0-9]+)", body).group(1)) / float(1000) + except: + pass + # Requests transforms any request into a GET after a redirect, # so the redirect has to be handled manually here to allow for # performing other types of requests even as the first request. @@ -105,10 +114,10 @@ def solve_cf_challenge(self, resp, **original_kwargs): cloudflare_kwargs["allow_redirects"] = False end_time = time.time() - time.sleep(self.delay - (end_time - start_time)) # Cloudflare requires a delay before solving the challenge + # Cloudflare requires a delay before solving the challenge + time.sleep(self.delay - (end_time - start_time)) redirect = self.request(method, submit_url, **cloudflare_kwargs) - redirect_location = urlparse(redirect.headers["Location"]) if not redirect_location.netloc: redirect_url = urlunparse((parsed_url.scheme, domain, redirect_location.path, redirect_location.params, redirect_location.query, redirect_location.fragment)) From 3c8fa45121ce4c94aa4fdf8629ccaea81811a9fc Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 5 Apr 2019 14:07:49 +0200 Subject: [PATCH 6/8] Make sure the headers are sent in proper order. --- cfscrape/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index 59dfb42..15260f3 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -64,6 +64,19 @@ def is_cloudflare_challenge(self, resp): ) def request(self, method, url, *args, **kwargs): + self.headers = ( + OrderedDict( + [ + ('User-Agent', self.headers['User-Agent']), + ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), + ('Accept-Language', 'en-US,en;q=0.5'), + ('Accept-Encoding', 'gzip, deflate'), + ('Connection', 'close'), + ('Upgrade-Insecure-Requests', '1') + ] + ) + ) + resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) # Check if Cloudflare anti-bot is on @@ -86,9 +99,9 @@ def solve_cf_challenge(self, resp, **original_kwargs): headers["Referer"] = resp.url try: + params["s"] = re.search(r'name="s"\svalue="(?P[^"]+)', body).group('s_value') params["jschl_vc"] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1) params["pass"] = re.search(r'name="pass" value="(.+?)"', body).group(1) - params["s"] = re.search(r'name="s"\svalue="(?P[^"]+)', body).group('s_value') except Exception as e: # Something is wrong with the page. # This may indicate Cloudflare has changed their anti-bot From e5c2292fc45ac8e955ef5ce3019809e30e4f6873 Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Fri, 5 Apr 2019 14:08:08 +0200 Subject: [PATCH 7/8] Define the new g variable. --- cfscrape/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index 15260f3..1bb8cbb 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -177,7 +177,7 @@ def solve_challenge(self, body, domain): js = '%s;%s;var t="%s";%s' % (simulate_document_js,atob_js,domain,js) buffer_js = "var Buffer = require('buffer').Buffer" # Pass Buffer into the new context, so it is available for atob. - js = "%s;console.log(require('vm').runInNewContext('%s', {'Buffer':Buffer}, {timeout: 5000}));" % (buffer_js, js) + js = "%s;console.log(require('vm').runInNewContext('%s', {'Buffer':Buffer,'g':String.fromCharCode}, {timeout: 5000}));" % (buffer_js, js) try: result = subprocess.check_output(["node", "-e", js]).strip() From 6538fb829c64b71a94223b1f86594cfb9dda16dc Mon Sep 17 00:00:00 2001 From: Lukas Pitschl Date: Wed, 10 Apr 2019 15:02:28 +0200 Subject: [PATCH 8/8] Avoid removing too much. --- cfscrape/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cfscrape/__init__.py b/cfscrape/__init__.py index 1bb8cbb..4749ad2 100644 --- a/cfscrape/__init__.py +++ b/cfscrape/__init__.py @@ -145,7 +145,14 @@ def solve_challenge(self, body, domain): raise ValueError("Unable to identify Cloudflare IUAM Javascript on website. %s" % BUG_REPORT) js = re.sub(r"a\.value = (.+\.toFixed\(10\);).+", r"\1", js) - js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js).replace("t.length", str(len(domain))) + # Match code that accesses the DOM and remove it, but without stripping too much. + try: + solution_name = re.search("s,t,o,p,b,r,e,a,k,i,n,g,f,\s*(.+)\s*=", js).groups(1) + match = re.search("(.*};)\n\s*(t\s*=(.+))\n\s*(;%s.*)" % (solution_name), js, re.M | re.I | re.DOTALL).groups() + js = match[0] + match[-1] + except Exception: + raise ValueError("Error parsing Cloudflare IUAM Javascript challenge. %s" % BUG_REPORT) + js = js.replace("t.length", str(len(domain))) # Strip characters that could be used to exit the string context # These characters are not currently used in Cloudflare's arithmetic snippet