Fix #364: Configure devbox in a vm

rsconf/component/__init__.py

@@ -315,7 +315,7 @@ def internal_build_write(self):
 
     def j2_ctx_init(self):
         self.j2_ctx = self.hdb.j2_ctx_copy()
-        d = self.j2_ctx.setdefault(self.name, PKDict())
+        d = self.j2_ctx.setdefault(self.module_name, PKDict())
         assert isinstance(
             d, PKDict
         ), f"component={self.name} is not a PKDict value={self.j2_ctx[self.name]}"

rsconf/component/devbox.py

@@ -29,8 +29,7 @@ def internal_build_compile(self):
                 )
             return
         self.buildt.require_component("docker", "network")
-        jc, _ = self.j2_ctx_init()
-        z = jc.devbox
+        jc, z = self.j2_ctx_init()
         z.setdefault("volumes", ["jupyter", "src"])
         z.host_d = systemd.unit_run_d(jc, self.name)
         self._gen_secrets(jc)

rsconf/component/network.py

@@ -97,7 +97,6 @@ def internal_build_compile(self):
                 ), "nat_input_dev and nat_output_dev both have to be defined"
 
     def internal_build_write(self):
-        jc = self.j2_ctx
         jc = self.j2_ctx
         z = jc.network
         # Order matters: _restricted_public_tcp_ports modifed public_tcp_ports

rsconf/component/vm_devbox.py

@@ -0,0 +1,63 @@
+"""Development in a vm
+
+:copyright: Copyright (c) 2023 RadiaSoft LLC.  All Rights Reserved.
+:license: http://www.apache.org/licenses/LICENSE-2.0.html
+"""
+from pykern import pkconfig
+from pykern import pkio
+from pykern.pkcollections import PKDict
+from pykern.pkdebug import pkdp
+from rsconf import component
+from rsconf import systemd
+
+_VM_DIR = "v"
+
+
+class T(component.T):
+    def internal_build_compile(self):
+        jc, z = self.j2_ctx_init()
+        if "user_name" not in self:
+            for u in self.hdb.vm_devbox.users:
+                self.buildt.build_component(
+                    T(
+                        f"{self.name}_{u}",
+                        self.buildt,
+                        user_name=u,
+                        module_name=self.name,
+                    )
+                )
+            return
+        self.buildt.require_component("network")
+        z.vm_d = systemd.custom_unit_prepare(self, self.j2_ctx).join(_VM_DIR)
+        z.ssh_port = jc.base_users.spec[self.user_name].vm_devbox_ssh_port
+        z.ssh_guest_host_key_f = "/etc/ssh/host_key"
+        z.ssh_guest_identity_pub_f = "/etc/ssh/identity.pub"
+        self._network(jc, z)
+        self._ssh(jc, z)
+
+    def internal_build_write(self):
+        jc = self.j2_ctx
+        if "user_name" not in self:
+            self.append_root_bash_with_main(jc)
+            return
+        z = jc[self.module_name]
+        systemd.install_unit_override(self, self.j2_ctx)
+        systemd.custom_unit_enable(
+            self, self.j2_ctx, run_u=jc.rsconf_db.run_u, run_group=jc.rsconf_db.run_u
+        )
+        self.install_access(mode="700", owner=jc.rsconf_db.run_u)
+        self.install_directory(z.vm_d)
+
+    def _network(self, jc, z):
+        n = self.buildt.get_component("network")
+        n.add_public_tcp_ports([str(z.ssh_port)])
+
+    def _ssh(self, jc, z):
+        z.sshd_config_f = z.vm_d.join("sshd_config")
+        s = super().gen_identity_and_host_ssh_keys(jc, "host", encrypt_identity=True)
+        z.pkupdate(
+            PKDict(
+                ssh_identity_pub_key=pkio.read_text(s["identity_pub_f"]),
+                ssh_host_key=pkio.read_text(s["host_key_f"]),
+            )
+        )

rsconf/package_data/vm_devbox/main.sh.jinja

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+vm_devbox_main() {
+    if vagrant --version > /dev/null 2>&1; then
+        return
+    fi
+    declare p=kernel-devel-$(uname -r)
+    if ! yum list "$p" &> /dev/null; then
+       install_err "rpm $p not found.
+Virtualbox needs the kernel-devel rpm for the host kernel to be installed.
+Maybe try updating the kernel? The repos only have kernel-devel for recent versions of the kernel."
+    fi
+    rsconf_yum_install "$p"
+    yum-config-manager --add-repo https://download.virtualbox.org/virtualbox/rpm/el/virtualbox.repo
+    yum makecache -y
+    rsconf_yum_install VirtualBox-7.0
+    yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+    rsconf_yum_install vagrant
+}

rsconf/package_data/vm_devbox/start.sh.jinja

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -eou pipefail
+
+vm_devbox_first_start() {
+    if [[ -e Vagrantfile ]]; then
+        return
+    fi
+    # Put vms in this dir so we don't scatter them around the machine.
+    # The default is under $HOME which doesn't make sense for this kind
+    # of service.
+    declare d=virtualbox_vms
+    mkdir "$d"
+    vboxmanage setproperty machinefolder "$PWD/$d"
+    curl https://radia.run | vagrant_dev_private_net= \
+        vagrant_dev_provision_eth1= \
+        vagrant_dev_no_mounts=1 \
+        vagrant_dev_no_nfs_src=1 \
+        vagrant_dev_no_vbguest=1 \
+        bash -s vagrant-sirepo-dev
+    vboxmanage setproperty machinefolder default
+}
+
+vm_devbox_set_forwarded_port() {
+    declare r=
+    if ! grep -q 'forwarded_port' Vagrantfile; then
+        perl -pi -e 's{^\s*(config.vm.hostname.*)$}{$1\nconfig.vm.network "forwarded_port", guest: {{ vm_devbox.ssh_port }}, host: {{ vm_devbox.ssh_port }}}' Vagrantfile
+        r=1
+    elif ! grep -q 'guest: {{ vm_devbox.ssh_port }}, host: {{ vm_devbox.ssh_port }}' Vagrantfile; then
+        perl -pi -e 's{^\s*config.vm.network "forwarded_port"}{config.vm.network "forwarded_port", guest: {{ vm_devbox.ssh_port }}, host: {{ vm_devbox.ssh_port }}}' Vagrantfile
+        r=1
+    fi
+    if [[ ${r:-} ]]; then
+        vagrant reload
+    fi
+}
+
+vm_devbox_set_ssh_config() {
+    if ! vagrant status | grep -q running; then
+        vagrant up
+    fi
+    vagrant ssh <<'EOF'
+sudo bash -s <<'EOF_BASH'
+set -eou pipefail
+
+install --mode=400 --owner=root --group=root /dev/stdin /etc/sshd_config<<EOF_INSTALL
+# DO NOT EDIT THIS FILE
+# MANAGED BY RSCONF
+
+# Keep vagrant ssh working
+Include /etc/ssh/sshd_config.d/*.conf
+ListenAddress 0.0.0.0:22
+
+HostKey {{ vm_devbox.ssh_guest_host_key_f }}
+ListenAddress 0.0.0.0:{{ vm_devbox.ssh_port }}
+
+AuthorizedKeysFile .ssh/authorized_keys
+PasswordAuthentication no
+PrintLastLog no
+Protocol 2
+X11Forwarding yes
+EOF_INSTALL
+
+echo '{{ vm_devbox.ssh_identity_pub_key }}' >> '/home/{{ rsconf_db.run_u }}/.ssh/authorized_keys'
+
+install --mode=400 --owner=root --group=root /dev/stdin "{{ vm_devbox.ssh_guest_host_key_f }}"<<EOF_INSTALL
+{{ vm_devbox.ssh_host_key }}
+EOF_INSTALL
+
+sshd -t
+systemctl restart sshd
+EOF_BASH
+EOF
+}
+
+cd {{ vm_devbox.vm_d }}
+vm_devbox_create_vm
+vm_devbox_set_forwarded_port
+vm_devbox_set_ssh_config

rsconf/package_data/vm_devbox/unit_override.conf.jinja

@@ -0,0 +1,4 @@
+[Service]
+RemainAfterExit=yes
+Restart=no
+Type=oneshot

rsconf/systemd.py

@@ -86,7 +86,7 @@ def custom_unit_enable(
 ):
     """Must be last call"""
     if not resource_d:
-        resource_d = compt.name
+        resource_d = compt.module_name
     z = j2_ctx.systemd
     z.update(
         after=_after(after),
@@ -254,7 +254,7 @@ def install_unit_override(compt, j2_ctx):
     compt.install_directory(d)
     compt.install_access(mode="444")
     compt.install_resource(
-        compt.name + "/unit_override.conf",
+        compt.module_name + "/unit_override.conf",
         j2_ctx,
         d.join("99-rsconf.conf"),
     )

tests/pkcli/build_data/1.in/db/000.yml

@@ -19,6 +19,7 @@ default:
       joeblow:
         uid: 2002
         email: devnull@v4.radia.run
+        vm_devbox_ssh_port: 102002
       marysmith:
         uid: 2003
         email: blackhole@v4.radia.run
@@ -557,6 +558,7 @@ host:
           - sirepo_job_supervisor
           - sirepo_jupyterhub
           - sirepo_test_http
+          - vm_devbox
       rsiviz:
         index_vhost: rsiviz.v9.radia.run
       sirepo:
@@ -603,6 +605,9 @@ host:
           v9.radia.run: sirepo.v9.radia.run
       sirepo_test_http:
         on_calendar: "9:00"
+      vm_devbox:
+        users:
+          - joeblow
 
 # testing named needs build-perl-rpms.sh
 #          - bivio_named

tests/pkcli/build_data/1.in/db/secret/v9.radia.run/vm_devbox/joeblow/host_key

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACCT/PsL9/XcOTmEXAvGVhJj5+MlLl/UBUNDiC+H84AAHQAAAJBriK60a4iu
+tAAAAAtzc2gtZWQyNTUxOQAAACCT/PsL9/XcOTmEXAvGVhJj5+MlLl/UBUNDiC+H84AAHQ
+AAAEBVP+MksqPH64Pi3Rfb8lieY9ofTRBWZqvKWk2oYjuqsJP8+wv39dw5OYRcC8ZWEmPn
+4yUuX9QFQ0OIL4fzgAAdAAAADHY5LnJhZGlhLnJ1bgE=
+-----END OPENSSH PRIVATE KEY-----

tests/pkcli/build_data/1.in/db/secret/v9.radia.run/vm_devbox/joeblow/host_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJP8+wv39dw5OYRcC8ZWEmPn4yUuX9QFQ0OIL4fzgAAd v9.radia.run

tests/pkcli/build_data/1.in/db/secret/v9.radia.run/vm_devbox/joeblow/identity

@@ -0,0 +1,8 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAmx+HJoQ
+g2f+QIRjDUx9W9AAAAEAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAINhimES0Iw3kh1fB
+0/VMNaXlwhWRDfEXt+3ZwEQNrvC9AAAAkKQ97+C2iZDRbWymbxMoPwqugDvdGtMAa2GmIg
+675vuEmzzETJcyz46jBqjgSMvaz2M0eF0o669pxO0tUC01YDSVXXcMQWoozwOgnwn9ljyl
+IOkUCH60EnKY4X7YgYEDUVG4ccfFiWNPggcD9H6y94TIbbd3MQ/7+he/9iVeuQFmmy/I5b
+qveSMj3DfuLReEnQ==
+-----END OPENSSH PRIVATE KEY-----

tests/pkcli/build_data/1.in/db/secret/v9.radia.run/vm_devbox/joeblow/identity.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKe3wWXD3GUVd/5viGVhf6L/ttJJfmksXauPSpC8zWru v9.radia.run

tests/pkcli/build_data/1.out/db/secret/v9.radia.run/vm_devbox/joeblow/identity

@@ -0,0 +1,8 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAmx+HJoQ
+g2f+QIRjDUx9W9AAAAEAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAINhimES0Iw3kh1fB
+0/VMNaXlwhWRDfEXt+3ZwEQNrvC9AAAAkKQ97+C2iZDRbWymbxMoPwqugDvdGtMAa2GmIg
+675vuEmzzETJcyz46jBqjgSMvaz2M0eF0o669pxO0tUC01YDSVXXcMQWoozwOgnwn9ljyl
+IOkUCH60EnKY4X7YgYEDUVG4ccfFiWNPggcD9H6y94TIbbd3MQ/7+he/9iVeuQFmmy/I5b
+qveSMj3DfuLReEnQ==
+-----END OPENSSH PRIVATE KEY-----

tests/pkcli/build_data/1.out/db/secret/v9.radia.run/vm_devbox/joeblow/identity.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKe3wWXD3GUVd/5viGVhf6L/ttJJfmksXauPSpC8zWru v9.radia.run

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh

@@ -35,3 +35,5 @@ rsconf_require sirepo_jupyterhub
 rsconf_require sirepo_job_supervisor
 rsconf_require sirepo
 rsconf_require sirepo_test_http
+rsconf_require vm_devbox_joeblow
+rsconf_require vm_devbox

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/etc/sysconfig/iptables

@@ -9,7 +9,7 @@
     -A INPUT -i lo -j ACCEPT
     -A INPUT -i em1 -j ACCEPT
 -A INPUT -i em2 -m state --state RELATED,ESTABLISHED -j ACCEPT
-    -A INPUT -i em2 -p tcp -m state --state NEW -m tcp --match multiport --dports 12000,12001,3100,3101,3102,3103,9999,http,pop3s,smtp,submission -j ACCEPT
+    -A INPUT -i em2 -p tcp -m state --state NEW -m tcp --match multiport --dports 102002,12000,12001,3100,3101,3102,3103,9999,http,pop3s,smtp,submission -j ACCEPT
         -A INPUT -i em2 -s 192.168.1.0/24 -p tcp -m state --state NEW -m tcp --dport https  -j ACCEPT
         -A INPUT -i em2 -s 127.0.0.1 -p tcp -m state --state NEW -m tcp --dport https  -j ACCEPT
 -A INPUT -i em2 -m state --state INVALID -j REJECT --reject-with icmp-port-unreachable

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/network.sh

@@ -6,7 +6,7 @@ rsconf_install_access '444' 'root' 'root'
 rsconf_install_file '/etc/resolv.conf' 'c333a7a816c03062d4e61effd6d2a8ce'
 rsconf_install_file '/etc/sysconfig/network-scripts/ifcfg-em1' 'e225a3f7b4b071e5c204b1182c17ab94'
 rsconf_install_file '/etc/sysconfig/network-scripts/ifcfg-em2' '4f8c01335a3bf1a8a89ba5d09fdf28df'
-rsconf_install_file '/etc/sysconfig/iptables' 'f47d00b954357d473fc5397dcf0a3d6f'
+rsconf_install_file '/etc/sysconfig/iptables' 'a6ab9a41a22c0f5fc2b43b523f97fdb4'
 network_main
 }
 #!/bin/bash

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/vm_devbox_joeblow/start

@@ -0,0 +1,86 @@
+#!/bin/bash
+set -eou pipefail
+
+vm_devbox_first_start() {
+    if [[ -e Vagrantfile ]]; then
+        return
+    fi
+    # Put vms in this dir so we don't scatter them around the machine.
+    # The default is under $HOME which doesn't make sense for this kind
+    # of service.
+    declare d=virtualbox_vms
+    mkdir "$d"
+    vboxmanage setproperty machinefolder "$PWD/$d"
+    curl https://radia.run | vagrant_dev_private_net= \
+        vagrant_dev_provision_eth1= \
+        vagrant_dev_no_mounts=1 \
+        vagrant_dev_no_nfs_src=1 \
+        vagrant_dev_no_vbguest=1 \
+        bash -s vagrant-sirepo-dev
+    vboxmanage setproperty machinefolder default
+}
+
+vm_devbox_set_forwarded_port() {
+    declare r=
+    if ! grep -q 'forwarded_port' Vagrantfile; then
+        perl -pi -e 's{^\s*(config.vm.hostname.*)$}{$1\nconfig.vm.network "forwarded_port", guest: 102002, host: 102002}' Vagrantfile
+        r=1
+    elif ! grep -q 'guest: 102002, host: 102002' Vagrantfile; then
+        perl -pi -e 's{^\s*config.vm.network "forwarded_port"}{config.vm.network "forwarded_port", guest: 102002, host: 102002}' Vagrantfile
+        r=1
+    fi
+    if [[ ${r:-} ]]; then
+        vagrant reload
+    fi
+}
+
+vm_devbox_set_ssh_config() {
+    if ! vagrant status | grep -q running; then
+        vagrant up
+    fi
+    vagrant ssh <<'EOF'
+sudo bash -s <<'EOF_BASH'
+set -eou pipefail
+
+install --mode=400 --owner=root --group=root /dev/stdin /etc/sshd_config<<EOF_INSTALL
+# DO NOT EDIT THIS FILE
+# MANAGED BY RSCONF
+
+# Keep vagrant ssh working
+Include /etc/ssh/sshd_config.d/*.conf
+ListenAddress 0.0.0.0:22
+
+HostKey /etc/ssh/host_key
+ListenAddress 0.0.0.0:102002
+
+AuthorizedKeysFile .ssh/authorized_keys
+PasswordAuthentication no
+PrintLastLog no
+Protocol 2
+X11Forwarding yes
+EOF_INSTALL
+
+echo 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKe3wWXD3GUVd/5viGVhf6L/ttJJfmksXauPSpC8zWru v9.radia.run
+' >> '/home/vagrant/.ssh/authorized_keys'
+
+install --mode=400 --owner=root --group=root /dev/stdin "/etc/ssh/host_key"<<EOF_INSTALL
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACCT/PsL9/XcOTmEXAvGVhJj5+MlLl/UBUNDiC+H84AAHQAAAJBriK60a4iu
+tAAAAAtzc2gtZWQyNTUxOQAAACCT/PsL9/XcOTmEXAvGVhJj5+MlLl/UBUNDiC+H84AAHQ
+AAAEBVP+MksqPH64Pi3Rfb8lieY9ofTRBWZqvKWk2oYjuqsJP8+wv39dw5OYRcC8ZWEmPn
+4yUuX9QFQ0OIL4fzgAAdAAAADHY5LnJhZGlhLnJ1bgE=
+-----END OPENSSH PRIVATE KEY-----
+
+EOF_INSTALL
+
+sshd -t
+systemctl restart sshd
+EOF_BASH
+EOF
+}
+
+cd /srv/vm_devbox_joeblow/v
+vm_devbox_create_vm
+vm_devbox_set_forwarded_port
+vm_devbox_set_ssh_config