fix #29 postfix gen files locally and refactor for two pass build

Fix #58 if a component defines internal_build_compile, internal_build_write
will be called later.
fix #28 sasldb can't be created on master, because berkley db files change each
time so create them on them on the host and move into place

rsconf/component/__init__.py

@@ -47,12 +47,20 @@ def assert_done(self):
         assert self.state == _DONE, \
             '{}: invalidate state for component.{}'.format(self.state, self.name)
 
-    def build(self):
+    def build_compile(self):
         self.state = _DONE
         self._install_access = pkcollections.Dict()
         self._root_bash = [self.name + _BASH_FUNC_SUFFIX + '() {']
         self._root_bash_aux = []
-        self.internal_build()
+        if hasattr(self, 'internal_build_compile'):
+            self.internal_build_compile()
+            self.buildt.append_write_queue(self)
+        else:
+            self.internal_build()
+            self.build_write()
+
+    def build_write(self):
+        self.internal_build_write()
         self.append_root_bash('}')
         self._root_bash.extend(self._root_bash_aux)
         self.buildt.write_root_bash(self.name, self._root_bash)
@@ -95,6 +103,17 @@ def install_directory(self, host_path):
     def install_ensure_file_exists(self, host_path):
         self._bash_append(host_path, is_file=True, ensure_exists=True)
 
+    def install_joined_lines(self, lines, host_path):
+        """Write lines with newlines to host_path
+
+        Args:
+            lines (object): iterable
+            host_path (py.path): where to install
+        """
+        p = self.tmp_path()
+        p.write(''.join([x + '\n' for x in lines]))
+        self.install_abspath(p, host_path)
+
     def install_perl_rpm(self, j2_ctx, rpm_base, channel=None):
         rpm_file = '{}-{}.rpm'.format(
             rpm_base,
@@ -144,6 +163,10 @@ def install_tls_key_and_crt(self, domain, dst_d):
         self.install_abspath(kc.crt, dst.crt, ignore_exists=True)
         return dst
 
+    def internal_build_write(self):
+        """Called after internal_build_compile"""
+        pass
+
     def rsconf_append(self, path, line_or_grep, line=None):
         l = "rsconf_edit_no_change_res=0 rsconf_append '{}' '{}'".format(path, line_or_grep)
         if not line is None:

rsconf/component/bop.py

@@ -43,6 +43,7 @@ def internal_build(self):
             j2_ctx = self.hdb.j2_ctx_copy()
             z = j2_ctx.bop
             z.mail_domain_keys = sorted(z.mail_domains.keys())
+            self.buildt.get_component('postfix').setup_bop(z.mail_domain_keys)
             z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE)
             nginx.update_j2_ctx_and_install_access(self, j2_ctx)
             self.install_resource(

rsconf/component/btest.py

@@ -34,6 +34,9 @@ def internal_build(self):
         )
         z.source_code_d = bop.SOURCE_CODE_D
         z.app_run_u = j2_ctx.rsconf_db.run_u
+        self.buildt.get_component('postfix').extend_local_host_names(
+            [z.client_host],
+        )
         self.install_access(mode='700', owner=z.app_run_u)
         self.install_directory(z.apps_d)
         z.home_d = db.user_home_path(j2_ctx, z.app_run_u)
@@ -106,4 +109,3 @@ def internal_build(self):
             j2_ctx,
             run_f,
         )
-        self.append_root_bash_with_main(j2_ctx)

rsconf/component/postfix.py

@@ -26,73 +26,94 @@
 
 class T(component.T):
 
-    def internal_build(self):
+    def extend_local_host_names(self, names):
+        self.j2_ctx.postfix.local_host_names.extend(names)
+
+    def internal_build_compile(self):
         from rsconf import systemd
 
         self.buildt.require_component('postgrey', 'spamd')
-        j2_ctx = self.hdb.j2_ctx_copy()
-        z = j2_ctx.setdefault('postfix', pkcollections.Dict())
+        self.j2_ctx = self.hdb.j2_ctx_copy()
+        jc = self.j2_ctx
+        z = jc.setdefault('postfix', pkcollections.Dict())
+        z.have_bop = False
         self.append_root_bash('rsconf_yum_install postfix procmail')
-        systemd.unit_prepare(self,j2_ctx, [_CONF_D])
-        self._setup_virtual_aliases(j2_ctx, z)
-        self._setup_sasl(j2_ctx, z)
-        self._setup_mynames(j2_ctx, z)
-        self._setup_check_sender_access(j2_ctx, z)
+        systemd.unit_prepare(self, jc, [_CONF_D])
+        self._setup_virtual_aliases(jc, z)
+        self._setup_sasl(jc, z)
+        self._setup_mynames(jc, z)
+        self._setup_check_sender_access(jc, z)
+        z.local_host_names = []
         z.local_host_names_f = '/etc/postfix/local-host-names'
-        # New install access
-        self.install_access(mode='400', owner=j2_ctx.rsconf_db.root_u)
-        kc = self.install_tls_key_and_crt(j2_ctx.rsconf_db.host, _CONF_D)
-        self.install_access(mode='644')
-        self.install_ensure_file_exists(z.local_host_names_f)
+
+    def internal_build_write(self):
+        from rsconf import systemd
+
+        jc = self.j2_ctx
+        z = jc.postfix
+        self.install_access(mode='400', owner=jc.rsconf_db.root_u)
+        kc = self.install_tls_key_and_crt(jc.rsconf_db.host, _CONF_D)
         z.update(
             tls_cert_file=kc.crt,
             tls_key_file=kc.key,
         )
+        self.install_access(mode='644')
+        self.install_resource('postfix/main.cf', jc, _CONF_D.join('main.cf'))
+        self.install_resource('postfix/master.cf', jc, _CONF_D.join('master.cf'))
+        self.install_joined_lines(
+            sorted(z.local_host_names),
+            z.local_host_names_f,
+        )
         # see base_users.py which may clear email_aliases by setting to None
-        z.aliases.update(j2_ctx.base_users.email_aliases)
+        z.aliases.update(jc.base_users.email_aliases)
         self.install_resource(
             'postfix/aliases',
-            j2_ctx,
+            jc,
             '/etc/aliases',
         )
-        self.append_root_bash_with_main(j2_ctx)
-        systemd.unit_enable(self, j2_ctx)
+        self.append_root_bash_with_main(jc)
+        systemd.unit_enable(self, jc)
         self.rsconf_service_restart_at_end()
 
-    def _setup_check_sender_access(self, j2_ctx, z):
+    def setup_bop(self, mail_domains):
+        self.j2_ctx.postfix.have_bop = True
+        self.extend_local_host_names(mail_domains)
+
+    def _setup_check_sender_access(self, jc, z):
         src = self.tmp_path()
         x = ['{} OK\n'.format(x) for x in sorted(z.get('whitelist_senders', []))]
         src.write(''.join(x))
         x =  _CONF_D.join('sender_access')
         z.check_sender_access_arg = 'texthash:' + str(x)
-        self.install_access(mode='440', owner=j2_ctx.rsconf_db.root_u, group='mail')
+        self.install_access(mode='440', owner=jc.rsconf_db.root_u, group='mail')
         self.install_abspath(src, x)
 
-    def _setup_mynames(self, j2_ctx, z):
+    def _setup_mynames(self, jc, z):
         if 'myhostname' in z:
             h = z.myhostname
-        elif 'primary_public_ip' in j2_ctx.network:
-            h = socket.gethostbyaddr(j2_ctx.network.primary_public_ip)[0]
+        elif 'primary_public_ip' in jc.network:
+            h = socket.gethostbyaddr(jc.network.primary_public_ip)[0]
             assert _HOSTNAME_RE.search(h).group(1).lower() \
-                == _HOSTNAME_RE.search(j2_ctx.rsconf_db.host).group(1).lower(), \
+                == _HOSTNAME_RE.search(jc.rsconf_db.host).group(1).lower(), \
                 '{}: reverse dns for {} does not match SLD of {}'.format(
                     h,
-                    j2_ctx.network.primary_public_ip,
-                    j2_ctx.rsconf_db.host,
+                    jc.network.primary_public_ip,
+                    jc.rsconf_db.host,
                 )
         else:
-            h = j2_ctx.rsconf_db.host
+            h = jc.rsconf_db.host
         # just in case
         h = h.lower()
         # allow overrides, but also assert "h"
         z.setdefault('mydomain', _HOSTNAME_RE.search(h).group(1))
         z.setdefault('myorigin', h)
         z.setdefault('myhostname', h)
 
-    def _setup_sasl(self, j2_ctx, z):
-        if not z.setdefault('sasl_users', []):
+    def _setup_sasl(self, jc, z):
+        z.have_sasl = bool(z.get('sasl_users'))
+        if not z.have_sasl:
             return
-        self.install_access(mode='400', owner=j2_ctx.rsconf_db.root_u)
+        self.install_access(mode='400', owner=jc.rsconf_db.root_u)
         z.sasl_users_flattened = []
         for domain, u in z.sasl_users.items():
             for user, password in u.items():
@@ -105,27 +126,26 @@ def _setup_sasl(self, j2_ctx, z):
                 )
         self.install_resource(
             'postfix/smtpd-sasldb.conf',
-            j2_ctx,
+            jc,
             '/etc/sasl2/smtpd-sasldb.conf',
         )
-        assert j2_ctx.postfix.sasl_users_flattened
-        self.hdb.postfix.sasl_users_flattened = j2_ctx.postfix.sasl_users_flattened
+        assert jc.postfix.sasl_users_flattened
+        self.hdb.postfix.sasl_users_flattened = jc.postfix.sasl_users_flattened
 
-    def _setup_virtual_aliases(self, j2_ctx, z):
-        if not z.setdefault('virtual_aliases', []):
+    def _setup_virtual_aliases(self, jc, z):
+        z.have_virtual_aliases = bool(z.get('virtual_aliases'))
+        if not z.have_virtual_aliases:
             return
-        self.install_access(mode='440', owner=j2_ctx.rsconf_db.root_u, group='mail')
+        self.install_access(mode='440', owner=jc.rsconf_db.root_u, group='mail')
         z.virtual_alias_f = _CONF_D.join('virtual_alias')
+        z.virtual_alias_maps = 'texthash:' + str(z.virtual_alias_f)
         z.virtual_alias_domains_f = _CONF_D.join('virtual_alias_domains')
-        self.install_resource(
-            'postfix/virtual_alias_domains', j2_ctx, z.virtual_alias_domains_f)
-        self.install_resource(
-            'postfix/virtual_alias', j2_ctx, z.virtual_alias_f)
-
-
-def sasl_user_password(j2_ctx, user):
-    # POSIT: user local part is globally unique
-    for u in j2_ctx.postfix.sasl_users_flattened:
-        if u.user == user:
-            return u.password
-    raise AssertionError('{}: not found in sasl users'.format(user))
+        domains = sorted(z.virtual_aliases.keys())
+        self.install_joined_lines(domains, z.virtual_alias_domains_f)
+        a = []
+        for d in domains:
+            a.extend([
+                '{}@{} {}'.format(x, d, z.virtual_aliases[d][x])
+                    for x in sorted(z.virtual_aliases[d].keys())
+            ])
+        self.install_joined_lines(a, z.virtual_alias_f)

rsconf/package_data/bop/main.sh.jinja

@@ -27,5 +27,4 @@ bop_facade_setup() {
 
 bop_main() {
     {{ bop.facade_setup_cmds | indent(4, indentfirst=False) }}
-    postfix_setup_bop '{{ bop.mail_domain_keys | join("' '") }}'
 }

rsconf/package_data/dev/db/000.yml.jinja

@@ -309,7 +309,7 @@ host:
         #components: [ rsconf, bkp, docker_registry, mpi_cluster ]
         # jupyterhub_proxy, btest, github_bkp, dovecot ]
         # components: [ rsconf, nfs_server, mpi_cluster ]
-        components: [ rsconf, docker_registry ]
+        components: [ rsconf, docker_registry, btest ]
     {{ host }}:
       docker:
         tls_host: {{ host }}
@@ -334,7 +334,8 @@ host:
         exports:
           /srv/sirepo/db/user: [ 10.10.10.0/24 ]
       rsconf_db:
-        components: [ jupyterhub, jupyterhub_proxy ]
+        components: [ bop ]
+#        components: [ jupyterhub, jupyterhub_proxy ]
 #        components: [ rabbitmq, celery_sirepo, sirepo, nfs_server ]
 #        components: [ nfs_client, docker ]
 #      sirepo:

rsconf/package_data/postfix/main.cf.jinja

@@ -0,0 +1,91 @@
+{#
+    Default config: grep -v '^#' /etc/postfix/main.cf
+    Then remove config lines that are also in Custom Config (below)
+#}
+queue_directory = /var/spool/postfix
+command_directory = /usr/sbin
+daemon_directory = /usr/libexec/postfix
+data_directory = /var/lib/postfix
+mail_owner = postfix
+inet_interfaces = all
+unknown_local_recipient_reject_code = 550
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+debug_peer_level = 2
+debugger_command =
+         PATH=/bin:/usr/bin:/usr/local/bin:/usr/X11R6/bin
+         ddd $daemon_directory/$process_name $process_id & sleep 5
+sendmail_path = /usr/sbin/sendmail.postfix
+newaliases_path = /usr/bin/newaliases.postfix
+mailq_path = /usr/bin/mailq.postfix
+setgid_group = postdrop
+html_directory = no
+manpage_directory = /usr/share/man
+sample_directory = /usr/share/doc/postfix-2.10.1/samples
+readme_directory = /usr/share/doc/postfix-2.10.1/README_FILES
+{#
+    Custom config: alphabetical
+#}
+{# https://help.ubuntu.com/community/PostfixGreylisting #}
+biff = no
+{# https://codinfox.github.io/dev/2015/04/08/postfix-cannot-start/ #}
+inet_protocols = ipv4
+mailbox_command = /usr/bin/procmail -t -Y -a "$EXTENSION" -d "$USER"
+local_recipient_maps =
+mailbox_size_limit = 0
+message_size_limit = 50000000
+mydestination = {{ postfix.mydomain }},localhost,{{ postfix.local_host_names_f }}
+mydomain = {{ postfix.mydomain }}
+myhostname = {{ postfix.myhostname }}
+mynetworks = 127.0.0.0/8
+myorigin = {{ postfix.myorigin }}
+smtp_tls_CAfile = /etc/pki/tls/certs/ca-bundle.crt
+smtp_tls_cert_file = {{ postfix.tls_cert_file }}
+smtp_tls_key_file = {{ postfix.tls_key_file }}
+smtp_tls_security_level = may
+smtpd_banner = {{ postfix.mydomain }} ESMTP
+smtpd_data_restrictions = reject_unauth_pipelining
+smtpd_recipient_restrictions = reject_unauth_pipelining,
+    permit_mynetworks,
+    permit_sasl_authenticated,
+    reject_unauth_destination,
+    reject_non_fqdn_recipient,
+    reject_unknown_recipient_domain,
+    check_policy_service inet:127.0.0.1:{{ postgrey.port }}
+smtpd_relay_restrictions = permit_mynetworks,permit_sasl_authenticated,reject_unauth_destination
+smtpd_tls_CAfile = /etc/pki/tls/certs/ca-bundle.crt
+smtpd_tls_cert_file = {{ postfix.tls_cert_file }}
+smtpd_tls_key_file = {{ postfix.tls_key_file }}
+smtpd_tls_security_level = may
+recipient_delimiter = +
+smtpd_sender_restrictions = reject_unauth_pipelining,
+    permit_mynetworks,
+    check_sender_access {{ postfix.check_sender_access_arg }},
+    reject_non_fqdn_sender,
+    reject_unknown_sender_domain
+{#
+    Too restrictive: host may not be valid
+    smtpd_helo_restrictions=reject_unknown_helo_hostname
+#}
+{#
+    Conditional Custom Config: alphabetical by conditional
+#}
+{% if postfix.have_bop %}
+mailbox_transport = bsendmailhttp:unix
+smtpd_timeout = ${stress?12}${stress:300}
+smtpd_hard_error_limit = ${stress?1}${stress:20}
+bsendmailhttp_destination_recipient_limit = 1
+{% endif %}
+{% if postfix.have_sasl %}
+smtpd_delay_reject = yes
+smtpd_sasl_path = smtpd-sasldb
+{% else %}
+smtpd_delay_reject = no
+{% endif %}
+{% if postfix.have_virtual_aliases %}
+virtual_alias_domains = {{ postfix.virtual_alias_domains_f }}
+virtual_alias_maps = {{ postfix.virtual_alias_maps }}
+{% endif %}
+{% if rsconf_db.channel != 'dev' %}
+smtpd_client_restrictions = sleep 8
+{% endif %}