[CT-352] catch and retry malformed json

core/dbt/clients/registry.py

@@ -1,7 +1,12 @@
 import functools
+import json
 import requests
 from dbt.events.functions import fire_event
-from dbt.events.types import RegistryProgressMakingGETRequest, RegistryProgressGETResponse
+from dbt.events.types import (
+    RegistryProgressMakingGETRequest,
+    RegistryProgressGETResponse,
+    RegistryMalformedResponse,
+)
 from dbt.utils import memoized, _connection_exception_retry as connection_exception_retry
 from dbt import deprecations
 import os
@@ -31,18 +36,34 @@ def _get(path, registry_base_url=None):
     fire_event(RegistryProgressGETResponse(url=url, resp_code=resp.status_code))
     resp.raise_for_status()
 
-    # It is unexpected for the content of the response to be None so if it is, raising this error
-    # will cause this function to retry (if called within _get_with_retries) and hopefully get
-    # a response.  This seems to happen when there's an issue with the Hub.
+    # It is unexpected for the content of the response to be None or malformed.  If it is,
+    # raising this error will cause this function to retry (if called within _get_with_retries)
+    # and hopefully get a valid response.  This seems to happen when there's an issue with the Hub.
     # See https://github.com/dbt-labs/dbt-core/issues/4577
-    if resp.json() is None:
+    # and https://github.com/dbt-labs/dbt-core/issues/4849
+    if not validJson(resp.json()):
         raise requests.exceptions.ContentDecodingError(
-            "Request error: The response is None", response=resp
+            "Request error: The response is not valid json", response=resp
         )
     return resp.json()
 
 
+def validJson(jsonData):
+    try:
+        # the quotes in the response were converted to single quotes which is not
+        # valid json.  json.dumps() will fix that.
+        json.loads(json.dumps(jsonData))
+    # We need to catch both malformed json and json = None
+    except (ValueError, TypeError) as exc:
+        # This event will allow us to see the actual json causing issues if more deps
+        # issues crop up.  Otherwise we're flying a little blind.
+        fire_event(RegistryMalformedResponse(jsonData=jsonData, exc=exc))
+        return False
+    return True
+
+
 def index(registry_base_url=None):
+    # this returns a list of all packages on the Hub
     return _get_with_retries("api/v1/index.json", registry_base_url)
 
 

core/dbt/events/types.py

@@ -2393,6 +2393,18 @@ def message(self) -> str:
         return f"External call exception: {self.exc}"
 
 
+@dataclass
+class RegistryMalformedResponse(DebugLevel):
+    jsonData: str
+    exc: Exception
+    code: str = "M022"
+
+    def message(self) -> str:
+        prefix = f"Malformed json: {self.jsonData}"
+        error = f"Exception: {self.exc}"
+        return f"{prefix}\n{error}"
+
+
 # since mypy doesn't run on every file we need to suggest to mypy that every
 # class gets instantiated. But we don't actually want to run this code.
 # making the conditional `if False` causes mypy to skip it as dead code so
@@ -2747,3 +2759,4 @@ def message(self) -> str:
     GeneralWarningException(exc=Exception(""), log_fmt="")
     EventBufferFull()
     RecordRetryException(exc=Exception(""))
+    RegistryMalformedResponse(jsonData="", exc=Exception(""))

test/unit/test_events.py

@@ -416,6 +416,7 @@ def MockNode():
     EventBufferFull(),
     RecordRetryException(exc=Exception('')),
     UnitTestInfo(msg=''),
+    RegistryMalformedResponse(jsonData="",exc=Exception('')),
 ]