Skip to content

Commit

Permalink
Feat/add try to datasette call (#330)
Browse files Browse the repository at this point in the history
* Added try loop and sleep around datasette download
  • Loading branch information
alexglasertpx authored Jan 29, 2025
1 parent 1c57cd1 commit d733222
Showing 1 changed file with 14 additions and 2 deletions.
16 changes: 14 additions & 2 deletions digital_land/expectations/operation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import pandas as pd
import urllib
import os
import time


# # TODO is there a way to represent this in a generalised count or not
Expand Down Expand Up @@ -39,7 +40,7 @@ def count_lpa_boundary(
lpa_geometry = data["geometry"]
except requests.exceptions.RequestException as err:
passed = False
message = f"An error occured when retrieving lpa geometry from platform {err}"
message = f"An error occurred when retrieving lpa geometry from platform {err}"
details = {}
return passed, message, details

Expand Down Expand Up @@ -142,7 +143,18 @@ def count_deleted_entities(
}
)
base_url = f"https://datasette.planning.data.gov.uk/digital-land.csv?{params}"
get_resource = pd.read_csv(base_url)

# Can have an issue getting data from datasette. If this occurs then wait a minute and retry
max_retries = 60 # Retry for an hour
for attempt in range(max_retries):
try:
get_resource = pd.read_csv(base_url)
break
except urllib.error.HTTPError:
time.sleep(60)
else:
raise Exception("Failed to fetch datasette after multiple attempts")

resource_list = get_resource["resource"].to_list()

# use resource list to get current entities
Expand Down

0 comments on commit d733222

Please sign in to comment.