Fix #496 Add pkcli.opendkim.gen_named_conf (#498)

* Fix #496 Add pkcli.opendkim.gen_named_conf
gen_key also calls gen_named_conf so always up to date
Removed named mode from component.opendkim

rsconf/component/opendkim.py

@@ -1,4 +1,4 @@
-"""create dkim configuration for named and/or opendkim
+"""create configuration for opendkim
 
 :copyright: Copyright (c) 2024 RadiaSoft LLC.  All Rights Reserved.
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
@@ -18,8 +18,6 @@
 class T(component.T):
     def internal_build_compile(self):
         jc, z = self.j2_ctx_init()
-        if self._named_compile(jc, z):
-            return
         self.buildt.require_component("postfix")
         self.append_root_bash("rsconf_yum_install opendkim")
         z.pksetdefault(port=8891, smtp_clients=[])
@@ -41,8 +39,6 @@ def internal_build_write(self):
 
         jc = self.j2_ctx
         z = jc[self.name]
-        if self._named_write(jc, z):
-            return
         systemd.unit_prepare(self, jc, watch_files=(_CONF_D, _CONF_F))
         self._install_conf(jc, z, self._install_keys(jc, z))
         systemd.unit_enable(self, jc)
@@ -96,7 +92,7 @@ def _iter_keys(self, z):
             yield d, z._keys[d]
 
     def _iter_key_rows(self, key):
-        for k in sorted(key.rows, key=lambda x: x.dns.subdomain):
+        for k in sorted(key.rows, key=lambda x: x.subdomain):
             yield k
 
     def _read_keys(self, jc, z):
@@ -120,52 +116,9 @@ def _find(secret_d):
         z._keys = _find(db.secret_path(jc, _SECRET_SUBDIR, visibility="global"))
         for d, v in z._keys.items():
             if not v.rows:
-                if not z.named_conf_d:
-                    raise AssertionError(f"missing keys for domain={d}")
-                v.rows.append(opendkim.gen_key(v.secret_d, d))
+                raise AssertionError(f"missing keys for domain={d}")
             for k in v.rows:
-                k.dns = opendkim.parse_txt(k.txt_f)
-
-    def _named_write(self, jc, z):
-        def _content():
-            return PKDict(
-                {d: _content_rows(s) for d, s in _normalize_domains().items()}
-            )
-
-        def _content_rows(subdomains):
-            rv = PKDict()
-            for s in subdomains:
-                for r in self._iter_key_rows(s.rows):
-                    rv[f"{r.dns.subdomain}{s.prefix}"] = r.dns.txt
-            return rv
-
-        def _normalize_domains():
-            rv = PKDict()
-            for d, v in self._iter_keys(z):
-                x = _split_domain(d)
-                rv.setdefault(x.sld, []).append(x.pkupdate(rows=v))
-            return rv
-
-        def _split_domain(full_name):
-            x = full_name.split(".")
-            return PKDict(
-                sld=".".join(x[-2:]),
-                prefix=".".join([""] + x[0:-2]) if len(x) > 2 else "",
-            )
-
-        if not z.named_conf_d:
-            return False
-        self.install_access(mode="400", owner=jc.rsconf_db.root_u)
-        self.install_json(_content(), z.named_conf_d.join("opendkim-named.json"))
-        return z.named_only
-
-    def _named_compile(self, jc, z):
-        z.pksetdefault(named_conf_d=None)
-        if not z.named_conf_d:
-            return False
-        z.pksetdefault(named_only=True)
-        self._read_keys(jc, z)
-        return z.named_only
+                k.subdomain = opendkim.public_key_info(k.txt_f)
 
     def _trusted_hosts(self, jc, z):
         from rsconf import db

rsconf/package_data/dev/db/secret/setup_dev.sh.jinja

@@ -4,27 +4,10 @@
 #
 set -euo pipefail
 
-_main() {
+
+_foss() {
     declare rpm_d=$1
     declare proprietary_d=$2
-    declare key_f='{{ rsconf_db.secret_d }}/bkp_ssh_key'
-    ssh-keygen -q -N '' -C '{{ master }}' -t ed25519 -f "$key_f"
-    # Key is manually installed
-    _root_install 700 -d /root/.ssh
-    _root_install 400 "$key_f" /root/.ssh/id_ed25519
-    rm -f "$key_f"
-    declare pub_key_f="$key_f.pub"
-    _root_install 400 "$pub_key_f" /root/.ssh/id_ed25519.pub
-    declare pub_key="$(cat "$pub_key_f")"
-    cat <<EOF > '{{ rsconf_db.secret_d }}/001.yml'
----
-default:
-  bkp:
-    ssh_key: "$pub_key"
-  rsconf:
-    ssh_key: "$pub_key"
-EOF
-    rm -f "$pub_key_f"
     declare f
     for f in bivio-perl perl-Bivio; do
         f=$f-dev.rpm
@@ -33,6 +16,22 @@ EOF
     echo 'created by setup_dev.sh.jinja for {{ all_host }} only' > "$proprietary_d/myapp-dev.tar.gz"
 }
 
+_main() {
+    declare rpm_d=$1
+    declare proprietary_d=$2
+    declare etc_d=$3
+    declare secret_d=$4
+    _ssh_keys
+    _foss "$rpm_d" "$proprietary_d"
+    _opendkim "$etc_d" "$secret_d"
+}
+
+_opendkim() {
+    declare etc_d=$1
+    declare secret_d=$2
+    rsconf opendkim gen_key "$secret_d/opendkim" "$etc_d/opendkim-named.json" '{{ host }}'
+}
+
 _root_install() {
     declare args=( "$@" )
     {% if unit_test %}
@@ -47,4 +46,25 @@ _root_install() {
     {% endif %}
 }
 
+_ssh_keys() {
+    declare key_f='{{ rsconf_db.secret_d }}/bkp_ssh_key'
+    ssh-keygen -q -N '' -C '{{ master }}' -t ed25519 -f "$key_f"
+    # Key is manually installed
+    _root_install 700 -d /root/.ssh
+    _root_install 400 "$key_f" /root/.ssh/id_ed25519
+    rm -f "$key_f"
+    declare pub_key_f="$key_f.pub"
+    _root_install 400 "$pub_key_f" /root/.ssh/id_ed25519.pub
+    declare pub_key="$(cat "$pub_key_f")"
+    cat <<EOF > '{{ rsconf_db.secret_d }}/001.yml'
+---
+default:
+  bkp:
+    ssh_key: "$pub_key"
+  rsconf:
+    ssh_key: "$pub_key"
+EOF
+    rm -f "$pub_key_f"
+}
+
 _main "$@"

rsconf/pkcli/opendkim.py

@@ -1,58 +1,71 @@
-"""generate and list keys
+"""Generate opendkim key pairs and opendkim-named.json
 
 :copyright: Copyright (c) 2024 RadiaSoft LLC.  All Rights Reserved.
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
 
 from pykern.pkcollections import PKDict
 from pykern.pkdebug import pkdc, pkdlog, pkdp
-import pykern.pkio
 import datetime
+import pykern.pkcli
+import pykern.pkio
+import pykern.pkjson
 import re
 import subprocess
 
 
-def gen_key(secret_dir, domain, selector=None):
-    """Generate opendkim key in db secret directory
+def gen_key(key_d, named_conf_f, domain, selector=None):
+    """Generate opendkim key pairs for domain in key_d
+
+    Creates key_d/domain/selector.{txt,private}. Calls
+    `gen_named_conf` when done.
 
     Args:
-        secret_dir (str): directory to write to
+        key_d (str): directory to write to
+        named_conf_f (str
         domain (str): domain to generate
+        selector (str): dkim selector for key [yyyymmdd]
     Returns:
         PKDict: private_f, txt_f, and selector
     """
-    rv = PKDict(
-        selector=selector or datetime.datetime.utcnow().strftime("%Y%m%d"),
-    )
-    p = pykern.pkio.mkdir_parent(pykern.pkio.py_path(secret_dir))
+    k = pykern.pkio.py_path(key_d)
     subprocess.check_call(
         (
             "opendkim-genkey",
             "-b",
             "2048",
             "-D",
-            str(p),
+            str(pykern.pkio.mkdir_parent(k.join(domain))),
             "-s",
-            rv.selector,
+            selector or datetime.datetime.utcnow().strftime("%Y%m%d"),
             "-d",
             domain,
         ),
         stderr=subprocess.STDOUT,
         shell=False,
     )
-    return rv.pkupdate(
-        private_f=p.join(f"{rv.selector}.private"),
-        txt_f=p.join(f"{rv.selector}.txt"),
-    )
+    gen_named_conf(k, named_conf_f)
 
 
-def parse_txt(path):
-    """Parse opendkim txt file
+def gen_named_conf(key_d, named_conf_f):
+    """Read key_d and write named_conf_f
+
+    key_d is created by `gen_key`. named_conf_f is a list of
+    domains which point to keys.
 
     Args:
-        path (str): txt file
+        key_d (str): dir that contains keys
+        named_conf_f (str): where to write json
+    """
+    pykern.pkjson.dump_pretty(_PublicKeys(key_d).as_dict(), filename=named_conf_f)
+
+
+def public_key_info(path):
+    """Parse DNS file (*.txt) to get key and selector
+
+
     Returns:
-        PKDict: subdomain and txt
+        PKDict: values selector and txt.
     """
     m = re.search(
         r'(\S+_domainkey).*\(\s*(".+")\s*\)',
@@ -62,6 +75,33 @@ def parse_txt(path):
     if not m:
         pykern.pkcli.command_error("path={} does not contain domainkey", path)
     return PKDict(
-        subdomain=m.group(1),
+        selector=m.group(1),
         txt=re.sub(r"\s+", " ", m.group(2), flags=re.DOTALL),
     )
+
+
+class _PublicKeys(PKDict):
+    """All keys in secret_d organized by second level domain (sld)
+
+    self index is sld, which contains a dict selector.subdomain
+    pointing to public key txt. The dictionaries are already sorted.
+    """
+
+    def __init__(self, key_d):
+        for p in pykern.pkio.walk_tree(key_d, file_re=r"\.txt$"):
+            self._append(
+                p.dirpath().basename,
+                public_key_info(p),
+            )
+
+    def as_dict(self):
+        def _content_rows(subdomains):
+            return PKDict({i.subdomain: i.txt for i in subdomains})
+
+        return PKDict({d: _content_rows(s) for d, s in self.items()})
+
+    def _append(self, domain, key_info):
+        x = domain.split(".")
+        key_info.sld = ".".join(x[-2:])
+        key_info.subdomain = ".".join([key_info.selector] + x[0:-2])
+        self.setdefault(key_info.sld, []).append(key_info)

rsconf/pkcli/setup_dev.py

@@ -88,10 +88,11 @@ def _sym(old, new_base=None):
         else:
             pkjinja.render_file(f, j2_ctx, output=dst, strict_undefined=True)
     n = []
-    for e in "rpm", "proprietary":
+    for e in "rpm", "proprietary", "etc":
         d = root_d.join(e)
         pkio.mkdir_parent(d)
         n.append(str(d))
+    n.append(str(secret_d))
     subprocess.check_call(
         ["bash", str(secret_d.join("setup_dev.sh")), *n],
     )

tests/pkcli/build1_data/1.in/db/000.yml

@@ -280,12 +280,9 @@ host:
                 mount_d: /srv/docker
       docker:
         tls_host: v3.radia.run
-      opendkim:
-        named_conf_d: /etc/named
       rsconf_db:
         components:
           - docker
-          - opendkim
           - rsconf
           - docker_registry
         local_dirs:

tests/pkcli/build1_data/1.out/srv/host/v3.radia.run/000.sh

@@ -7,7 +7,6 @@ rsconf_require logrotate
 rsconf_require base_all
 rsconf_require db_bkp
 rsconf_require docker
-rsconf_require opendkim
 rsconf_require nginx
 rsconf_require rsconf
 rsconf_require docker_registry

tests/pkcli/opendkim_test.py

@@ -0,0 +1,26 @@
+"""test pkcli opendkim
+
+:copyright: Copyright (c) 2024 RadiaSoft LLC.  All Rights Reserved.
+:license: http://www.apache.org/licenses/LICENSE-2.0.html
+"""
+
+
+def test_gen_key():
+    from pykern import pkunit, pkio, pkjson
+    from pykern.pkdebug import pkdlog, pkdp
+    from rsconf.pkcli import opendkim
+
+    d = pkunit.empty_work_dir()
+    n = d.join("opendkim-named.json")
+    k = d.join("keys")
+    opendkim.gen_key(k, n, "b.c", selector="s1")
+    a = pkjson.load_any(pkio.read_text(n))
+    pkunit.pkeq(["b.c"], list(a.keys()))
+    pkunit.pkeq(["s1._domainkey"], list(a["b.c"].keys()))
+    pkunit.pkre(r'^"v=DKIM.*\w"$', a["b.c"]["s1._domainkey"])
+    opendkim.gen_key(k, n, "a.b.c", selector="p1")
+    opendkim.gen_key(k, n, "a.a.a", selector="s3")
+    a = pkjson.load_any(pkio.read_text(n))
+    pkunit.pkeq(["a.a", "b.c"], list(a.keys()))
+    pkunit.pkeq(["p1._domainkey.a", "s1._domainkey"], list(a["b.c"].keys()))
+    pkunit.pkeq(["s3._domainkey.a"], list(a["a.a"].keys()))