Skip to content

Commit

Permalink
Add workaround for negative page load timeouts
Browse files Browse the repository at this point in the history
When this happens, driver.current_url is
chrome-extension://.../skin/options.html

Seems like we should be able to retry visiting the site.
  • Loading branch information
ghostwords committed Jan 6, 2025
1 parent e4b145d commit 1d297fc
Showing 1 changed file with 10 additions and 5 deletions.
15 changes: 10 additions & 5 deletions crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -854,7 +854,15 @@ def visit_domain(self, domain):
Visit a domain, then spend `self.wait_time` seconds on the site
waiting for dynamic loading to complete.
"""
self.handle_alerts_and(lambda: self.driver.get(f"http://{domain}/"))
for i in range(3):
try:
self.handle_alerts_and(lambda: self.driver.get(f"http://{domain}/"))
except TimeoutException as ex:
if str(ex).startswith("Timed out receiving message from renderer: -"):
self.logger.info("Timed out with negative value, retrying...")
time.sleep(2 + i)
else:
raise

self.raise_on_chrome_error_pages()
self.raise_on_security_pages()
Expand Down Expand Up @@ -998,10 +1006,7 @@ def log_snitch_map_changes(self, old_snitches, new_snitches):
def get_current_url(self):
for i in range(3):
try:
curl = self.driver.current_url
if curl.startswith("chrome-extension://"):
self.logger.error(traceback.format_exc())
return curl
return self.driver.current_url
except TimeoutException:
self.logger.info("Timed out getting driver.current_url, retrying...")
time.sleep(2 + i)
Expand Down

0 comments on commit 1d297fc

Please sign in to comment.