Skip to content

Commit

Permalink
Don't look up driver.current_url on exceptions
Browse files Browse the repository at this point in the history
It doesn't seem worth the complexity.
  • Loading branch information
ghostwords committed Jan 8, 2025
1 parent 3f16ada commit d029d68
Showing 1 changed file with 8 additions and 13 deletions.
21 changes: 8 additions & 13 deletions crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,20 +610,19 @@ def _load_ext_page():
try:
self.handle_alerts_and(_load_ext_page)
break
except (MaxRetryError, ProtocolError, ReadTimeoutError) as e:
self.logger.warning("Error loading extension page:\n%s", str(e))
except (MaxRetryError, ProtocolError, ReadTimeoutError) as ex:
self.logger.error("Error loading extension page:\n%s", str(ex))
self.restart_browser()
except TimeoutException:
num_timeouts += 1
self.logger.warning("Timed out loading extension page")
if num_timeouts >= max_timeouts:
num_timeouts = 0
self.restart_browser()
except WebDriverException as err:
self.logger.warning(
"Error loading extension page (%s): %s",
type(err).__name__, err.msg)
if should_restart(err):
except WebDriverException as ex:
self.logger.error("Error loading extension page (%s): %s",
type(ex).__name__, ex.msg)
if should_restart(ex):
self.restart_browser()
else:
raise WebDriverException("Failed to load extension page")
Expand Down Expand Up @@ -1051,14 +1050,10 @@ def crawl(self, domains):
self.logger.warning("Timed out loading %s", curl or domain)

except WebDriverException as ex:
self.logger.error("%s on %s: %s",
type(ex).__name__, domain, ex.msg)
if should_restart(ex):
self.logger.error("%s on %s: %s", type(ex).__name__,
domain, ex.msg)
self.restart_browser()
else:
self.logger.error("%s on %s: %s", type(ex).__name__,
self.get_current_url() or domain,
ex.msg)

num_total = len(domains)
if num_total:
Expand Down

0 comments on commit d029d68

Please sign in to comment.