From 727a5f881c644a70cd82353eb6697a0d803e619a Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 21 Feb 2025 16:45:02 +0100 Subject: [PATCH 1/4] feat(certs): capture ACME error message Reduce ACME job timeout from 120 to 30 seconds. --- .../actions/delete-certificate/20writeconfig | 8 ++++++ .../actions/set-certificate/20writeconfig | 8 +++++- imageroot/pypkg/cert_helpers.py | 27 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/imageroot/actions/delete-certificate/20writeconfig b/imageroot/actions/delete-certificate/20writeconfig index 11990b0..3c56b72 100755 --- a/imageroot/actions/delete-certificate/20writeconfig +++ b/imageroot/actions/delete-certificate/20writeconfig @@ -10,14 +10,22 @@ import sys import os import cert_helpers import agent +import datetime def main(): + tstart = datetime.datetime.now(datetime.UTC) request = json.load(sys.stdin) fqdn = request['fqdn'] if fqdn in cert_helpers.read_custom_cert_names(): cert_helpers.remove_custom_cert(fqdn) elif fqdn in cert_helpers.read_default_cert_names(): cert_helpers.remove_default_certificate_name(fqdn) + obtained = cert_helpers.wait_acmejson_sync(timeout=request.get('sync_timeout', 30)) + if not obtained: + acme_error = cert_helpers.traefik_last_acme_error_since(tstart) + for errline in acme_error.split("\n"): + print(agent.SD_ERR + errline, file=sys.stderr) + exit(3) else: agent.set_status('validation-failed') json.dump([{'field': 'fqdn','parameter':'fqdn','value': fqdn,'error':'certificate_not_found'}], fp=sys.stdout) diff --git a/imageroot/actions/set-certificate/20writeconfig b/imageroot/actions/set-certificate/20writeconfig index 3a81248..c6d0173 100755 --- a/imageroot/actions/set-certificate/20writeconfig +++ b/imageroot/actions/set-certificate/20writeconfig @@ -9,16 +9,22 @@ import json import sys import os import cert_helpers +import agent +import datetime def main(): + tstart = datetime.datetime.now(datetime.UTC) request = json.load(sys.stdin) cert_helpers.add_default_certificate_name(request['fqdn']) if request.get('sync'): - obtained = cert_helpers.wait_acmejson_sync(timeout=request.get('sync_timeout', 120)) + obtained = cert_helpers.wait_acmejson_sync(timeout=request.get('sync_timeout', 30)) else: obtained = False json.dump({"obtained": obtained}, fp=sys.stdout) if request.get('sync') is not None and obtained is False: + acme_error = cert_helpers.traefik_last_acme_error_since(tstart) + for errline in acme_error.split('\n'): + print(agent.SD_ERR + errline, file=sys.stderr) exit(2) if __name__ == "__main__": diff --git a/imageroot/pypkg/cert_helpers.py b/imageroot/pypkg/cert_helpers.py index 069280c..44ab983 100644 --- a/imageroot/pypkg/cert_helpers.py +++ b/imageroot/pypkg/cert_helpers.py @@ -10,6 +10,8 @@ import json import time import glob +import subprocess +import datetime def read_default_cert_names(): """Return the list of host names configured in the @@ -165,3 +167,28 @@ def parse_yaml_config(path): with open(path, 'r') as fp: conf = yaml.safe_load(fp) return conf + +def traefik_last_acme_error_since(tstart): + """Get the last Traefik error related to ACME from Loki. + + :param tstart: a ISO8601 string with TZ offset + :return: string + """ + try: + acme_error = subprocess.check_output([ + "logcli", + "query", + "--limit=1", + "--from=" + tstart.isoformat(), + "--timezone=Local", # use system timezone for output + "--quiet", + "--no-labels", + '{module_id=~"traefik.+"} | json | line_format "{{.MESSAGE}}"' + \ + '| logfmt | providerName="acmeServer.acme" and error!=""' + \ + '| line_format "{{.error}}"', + ], timeout=15, text=True) + except subprocess.TimeoutExpired as ex: + acme_error = 'traefik_last_acme_error_since(): logcli timeout - ' + str(ex) + except subprocess.CalledProcessError as ex: + acme_error = 'traefik_last_acme_error_since(): logcli error - ' + str(ex) + return acme_error From fa346bf998b57a2978f901caf68c311be84f231b Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 21 Feb 2025 17:10:56 +0100 Subject: [PATCH 2/4] fix: wait on self-signed cert --- imageroot/pypkg/cert_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imageroot/pypkg/cert_helpers.py b/imageroot/pypkg/cert_helpers.py index 44ab983..c5449e9 100644 --- a/imageroot/pypkg/cert_helpers.py +++ b/imageroot/pypkg/cert_helpers.py @@ -79,6 +79,8 @@ def wait_acmejson_sync(timeout=120, interval=2.1, names=[]): if not names: # Wait for the default certificate. names = read_default_cert_names() + if not names: + return True # Consider as obtained, if no names are set. elapsed = 0.0 while elapsed < timeout: time.sleep(interval) From 88d2eed68a024febefc4437d9d413db38a9ed5d3 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 21 Feb 2025 17:19:52 +0100 Subject: [PATCH 3/4] fix: handle acme.json special cases --- imageroot/pypkg/cert_helpers.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/imageroot/pypkg/cert_helpers.py b/imageroot/pypkg/cert_helpers.py index c5449e9..5b6adc7 100644 --- a/imageroot/pypkg/cert_helpers.py +++ b/imageroot/pypkg/cert_helpers.py @@ -53,21 +53,28 @@ def remove_custom_cert(name): def has_acmejson_name(name): """Return True if name is found among acme.json Certificates.""" - with open('acme/acme.json', 'r') as fp: - acmejson = json.load(fp) - for ocert in acmejson['acmeServer']["Certificates"] or []: - if ocert["domain"]["main"] == name or name in ocert["domain"].get("sans", []): - return True + try: + with open('acme/acme.json', 'r') as fp: + acmejson = json.load(fp) + for ocert in acmejson['acmeServer']["Certificates"] or []: + if ocert["domain"]["main"] == name or name in ocert["domain"].get("sans", []): + return True + except (FileNotFoundError, KeyError): + pass return False def has_acmejson_cert(main, sans=[]): """Return True if a certificate matching main and sans is found among acme.json Certificates.""" - with open('acme/acme.json', 'r') as fp: - acmejson = json.load(fp) - for ocert in acmejson['acmeServer']["Certificates"] or []: - if ocert["domain"]["main"] == main and set(ocert["domain"].get("sans", [])) == set(sans): - return True + try: + with open('acme/acme.json', 'r') as fp: + acmejson = json.load(fp) + for ocert in acmejson['acmeServer']["Certificates"] or []: + if ocert["domain"]["main"] == main and set(ocert["domain"].get("sans", [])) == set(sans): + return True + return False + except (FileNotFoundError, KeyError): + pass return False def wait_acmejson_sync(timeout=120, interval=2.1, names=[]): From 06e2003c5443f552497bc14c06c9090e185137da Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Mon, 24 Feb 2025 11:44:20 +0100 Subject: [PATCH 4/4] fixup! feat(certs): capture ACME error message --- imageroot/actions/create-module/10expandconfig | 1 + imageroot/update-module.d/15upgrade_v3 | 1 + 2 files changed, 2 insertions(+) diff --git a/imageroot/actions/create-module/10expandconfig b/imageroot/actions/create-module/10expandconfig index d79d092..9f20935 100755 --- a/imageroot/actions/create-module/10expandconfig +++ b/imageroot/actions/create-module/10expandconfig @@ -19,6 +19,7 @@ file: {} log: level: INFO + noColor: true accessLog: {} diff --git a/imageroot/update-module.d/15upgrade_v3 b/imageroot/update-module.d/15upgrade_v3 index 91514ac..3a35d20 100755 --- a/imageroot/update-module.d/15upgrade_v3 +++ b/imageroot/update-module.d/15upgrade_v3 @@ -48,6 +48,7 @@ def upgrade_to_v3(static_cfg): write_dynamic_config("./configs/_default_cert.yml", defcert) # Required configuration upgrade from v2 to v3 format: static_cfg['core']['defaultRuleSyntax'] = 'v3' + static_cfg['log'] = {'noColor': True, 'level': 'INFO'} for config_file in glob.glob("./configs/*.yml"): try: with open(config_file) as ofile: