feat(certs): capture ACME error message

imageroot/actions/delete-certificate/20writeconfig

@@ -10,14 +10,22 @@ import sys
 import os
 import cert_helpers
 import agent
+import datetime
 
 def main():
+    tstart = datetime.datetime.now(datetime.UTC)
     request = json.load(sys.stdin)
     fqdn = request['fqdn']
     if fqdn in cert_helpers.read_custom_cert_names():
         cert_helpers.remove_custom_cert(fqdn)
     elif fqdn in cert_helpers.read_default_cert_names():
         cert_helpers.remove_default_certificate_name(fqdn)
+        obtained = cert_helpers.wait_acmejson_sync(timeout=request.get('sync_timeout', 30))
+        if not obtained:
+            acme_error = cert_helpers.traefik_last_acme_error_since(tstart)
+            for errline in acme_error.split("\n"):
+                print(agent.SD_ERR + errline, file=sys.stderr)
+            exit(3)
     else:
         agent.set_status('validation-failed')
         json.dump([{'field': 'fqdn','parameter':'fqdn','value': fqdn,'error':'certificate_not_found'}], fp=sys.stdout)

imageroot/actions/set-certificate/20writeconfig

@@ -9,16 +9,22 @@ import json
 import sys
 import os
 import cert_helpers
+import agent
+import datetime
 
 def main():
+    tstart = datetime.datetime.now(datetime.UTC)
     request = json.load(sys.stdin)
     cert_helpers.add_default_certificate_name(request['fqdn'])
     if request.get('sync'):
-        obtained = cert_helpers.wait_acmejson_sync(timeout=request.get('sync_timeout', 120))
+        obtained = cert_helpers.wait_acmejson_sync(timeout=request.get('sync_timeout', 30))
     else:
         obtained = False
     json.dump({"obtained": obtained}, fp=sys.stdout)
     if request.get('sync') is not None and obtained is False:
+        acme_error = cert_helpers.traefik_last_acme_error_since(tstart)
+        for errline in acme_error.split('\n'):
+            print(agent.SD_ERR + errline, file=sys.stderr)
         exit(2)
 
 if __name__ == "__main__":

imageroot/pypkg/cert_helpers.py

@@ -10,6 +10,8 @@
 import json
 import time
 import glob
+import subprocess
+import datetime
 
 def read_default_cert_names():
     """Return the list of host names configured in the
@@ -51,21 +53,28 @@ def remove_custom_cert(name):
 
 def has_acmejson_name(name):
     """Return True if name is found among acme.json Certificates."""
-    with open('acme/acme.json', 'r') as fp:
-        acmejson = json.load(fp)
-    for ocert in acmejson['acmeServer']["Certificates"] or []:
-        if ocert["domain"]["main"] == name or name in ocert["domain"].get("sans", []):
-            return True
+    try:
+        with open('acme/acme.json', 'r') as fp:
+            acmejson = json.load(fp)
+        for ocert in acmejson['acmeServer']["Certificates"] or []:
+            if ocert["domain"]["main"] == name or name in ocert["domain"].get("sans", []):
+                return True
+    except (FileNotFoundError, KeyError):
+        pass
     return False
 
 def has_acmejson_cert(main, sans=[]):
     """Return True if a certificate matching main and sans is found among
     acme.json Certificates."""
-    with open('acme/acme.json', 'r') as fp:
-        acmejson = json.load(fp)
-    for ocert in acmejson['acmeServer']["Certificates"] or []:
-        if ocert["domain"]["main"] == main and set(ocert["domain"].get("sans", [])) == set(sans):
-            return True
+    try:
+        with open('acme/acme.json', 'r') as fp:
+            acmejson = json.load(fp)
+        for ocert in acmejson['acmeServer']["Certificates"] or []:
+            if ocert["domain"]["main"] == main and set(ocert["domain"].get("sans", [])) == set(sans):
+                return True
+        return False
+    except (FileNotFoundError, KeyError):
+        pass
     return False
 
 def wait_acmejson_sync(timeout=120, interval=2.1, names=[]):
@@ -77,6 +86,8 @@ def wait_acmejson_sync(timeout=120, interval=2.1, names=[]):
     if not names:
         # Wait for the default certificate.
         names = read_default_cert_names()
+    if not names:
+        return True # Consider as obtained, if no names are set.
     elapsed = 0.0
     while elapsed < timeout:
         time.sleep(interval)
@@ -165,3 +176,28 @@ def parse_yaml_config(path):
     with open(path, 'r') as fp:
         conf = yaml.safe_load(fp)
     return conf
+
+def traefik_last_acme_error_since(tstart):
+    """Get the last Traefik error related to ACME from Loki.
+
+    :param tstart: a ISO8601 string with TZ offset
+    :return: string
+    """
+    try:
+        acme_error = subprocess.check_output([
+            "logcli",
+            "query",
+            "--limit=1",
+            "--from=" + tstart.isoformat(),
+            "--timezone=Local", # use system timezone for output
+            "--quiet",
+            "--no-labels",
+            '{module_id=~"traefik.+"} | json | line_format "{{.MESSAGE}}"' + \
+            '| logfmt | providerName="acmeServer.acme" and error!=""' + \
+            '| line_format "{{.error}}"',
+        ], timeout=15, text=True)
+    except subprocess.TimeoutExpired as ex:
+        acme_error = 'traefik_last_acme_error_since(): logcli timeout - ' + str(ex)
+    except subprocess.CalledProcessError as ex:
+        acme_error = 'traefik_last_acme_error_since(): logcli error - ' + str(ex)
+    return acme_error