diff --git a/core/dbt/clients/registry.py b/core/dbt/clients/registry.py index 846b4f622c8..c0687099f48 100644 --- a/core/dbt/clients/registry.py +++ b/core/dbt/clients/registry.py @@ -20,7 +20,7 @@ def _get_url(url, registry_base_url=None): def _get_with_retries(path, registry_base_url=None): - get_fn = functools.partial(_get, path, registry_base_url) + get_fn = functools.partial(_get_cached, path, registry_base_url) return connection_exception_retry(get_fn, 5) @@ -31,28 +31,31 @@ def _get(path, registry_base_url=None): fire_event(RegistryProgressGETResponse(url=url, resp_code=resp.status_code)) resp.raise_for_status() + json_response = ( + resp.json() # if this fails, it should raise requests.exceptions.JSONDecodeError and trigger retry + ) # It is unexpected for the content of the response to be None so if it is, raising this error # will cause this function to retry (if called within _get_with_retries) and hopefully get # a response. This seems to happen when there's an issue with the Hub. # See https://github.com/dbt-labs/dbt-core/issues/4577 - if resp.json() is None: + if json_response is None: raise requests.exceptions.ContentDecodingError( "Request error: The response is None", response=resp ) - return resp.json() + return json_response + + +_get_cached = memoized(_get) def index(registry_base_url=None): return _get_with_retries("api/v1/index.json", registry_base_url) +# is this redundant, now that all _get responses are being cached? index_cached = memoized(index) -def packages(registry_base_url=None): - return _get_with_retries("api/v1/packages.json", registry_base_url) - - def package(name, registry_base_url=None): response = _get_with_retries("api/v1/{}.json".format(name), registry_base_url) @@ -80,7 +83,8 @@ def package(name, registry_base_url=None): def package_version(name, version, registry_base_url=None): - return _get_with_retries("api/v1/{}/{}.json".format(name, version), registry_base_url) + response = package(name, registry_base_url) + return response["versions"][version] def get_available_versions(name): diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 5190a58f210..870d8e06a90 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2422,6 +2422,8 @@ def message(self) -> str: GitNothingToDo(sha="") GitProgressUpdatedCheckoutRange(start_sha="", end_sha="") GitProgressCheckedOutAt(end_sha="") + RegistryProgressMakingGETRequest(url="") + RegistryProgressGETResponse(url="", resp_code=1234) SystemErrorRetrievingModTime(path="") SystemCouldNotWrite(path="", reason="", exc=Exception("")) SystemExecutingCmd(cmd=[""])