Add fsck to dracut modules

packages/base-dracut-modules/definition.yaml

@@ -1,4 +1,4 @@
 name: "base-dracut-modules"
 category: "system"
-version: 0.1.1-29
+version: "0.1.2"
 description: "Base modules for creating an initrd with dracut for cOS derivatives"

packages/cos/collection.yaml

@@ -2,20 +2,18 @@ packages:
   - &cos
     name: "cos"
     category: "system"
-    version: 0.8.11-2
+    version: "0.8.12"
     description: "cOS base image, used to build cOS live ISOs"
     brand_name: "cOS"
     labels:
       autobump.revdeps: "true"
   - !!merge <<: *cos
     name: "cos-container"
     description: "cOS container image, used to build cOS derivatives from scratch"
-    version: 0.8.11-3
   - !!merge <<: *cos
     category: "recovery"
     brand_name: "cOS recovery"
     description: "cOS recovery image, used to boot cOS for troubleshooting"
-    version: 0.8.11-2
   - !!merge <<: *cos
     name: "cos-img"
     category: "recovery"

packages/grub2/collection.yaml

@@ -7,7 +7,7 @@ packages:
         version: ">0.0.2"
   - name: "grub2-config"
     category: "system"
-    version: 0.0.16-1
+    version: "0.0.17"
     provides:
       - name: "grub-config"
         version: ">0.0.12"

packages/grub2/config/bootargs.cfg.tmpl

@@ -2,7 +2,7 @@ set kernel=/boot/vmlinuz
 if [ -n "$recoverylabel" ]; then
     set kernelcmd="console=tty1 console=ttyS0 root=live:LABEL=$recoverylabel rd.live.dir=/ rd.live.squashimg=$img panic=5 rd.neednet=1 rd.cos.oemlabel=@OEM_LABEL@"
 else
-    set kernelcmd="console=tty1 console=ttyS0 root=LABEL=$label cos-img/filename=$img panic=5 security=selinux selinux=1 rd.neednet=1 rd.cos.oemlabel=@OEM_LABEL@"
+    set kernelcmd="console=tty1 console=ttyS0 root=LABEL=$label cos-img/filename=$img panic=5 security=selinux selinux=1 rd.neednet=1 rd.cos.oemlabel=@OEM_LABEL@ fsck.mode=force fsck.repair=yes"
 fi
 
 set initramfs=/boot/initrd

packages/immutable-rootfs/30cos-immutable-rootfs/cos-loop-img.sh

@@ -9,16 +9,38 @@ function doLoopMount {
         label=$(basename "${dev}")
         [ -e "/tmp/cosloop-${label}" ] && continue
         > "/tmp/cosloop-${label}" 
+
         mount -t auto -o "${cos_root_perm}" "/dev/disk/by-label/${label}" "${cos_state}" || continue
         if [ -f "${cos_state}/${cos_img}" ]; then
-            losetup -f "${cos_state}/${cos_img}"
+
+            # FSCHECK if cos_root_perm == "ro" on both
+            if [ "$cos_root_perm" == "ro" ]; then
+               systemd-fsck "/dev/disk/by-label/${label}"
+            fi
+
+            dev=$(losetup --show -f "${cos_state}/${cos_img}")
+
+            # FSCHECK if cos_root_perm == "ro"
+            if [ "$cos_root_perm" == "ro" ]; then
+               systemd-fsck "$dev"
+            fi
+
             exit 0
         else
             umount "${cos_state}"
         fi
     done
 }
 
+function dofsCheck {
+    # Iterate over current partitions
+    # As fs corruption could lead to partitions with no label, we scan here for all partitions found and we run systemd-fsck
+    for dev in /dev/disk/by-partuuid/*; do
+        partuuid=$(basename "${dev}")
+        systemd-fsck "/dev/disk/by-partuuid/${partuuid}"
+    done
+}
+
 type getarg > /dev/null 2>&1 || . /lib/dracut-lib.sh
 
 PATH=/usr/sbin:/usr/bin:/sbin:/bin
@@ -37,6 +59,7 @@ ismounted "${cos_state}" && exit 0
 
 mkdir -p "${cos_state}"
 
+dofsCheck
 doLoopMount
 
 rm -r "${cos_state}"

packages/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh

@@ -248,6 +248,8 @@ for mount in "${mountpoints[@]}"; do
             fstab+=$(mountOverlay "${mount%%:*}")
         fi
     else
+        # FSCK
+        systemd-fsck "${mount}"
         fstab+=$(mountPersistent "${mount}")
     fi
 done

packages/immutable-rootfs/30cos-immutable-rootfs/module-setup.sh

@@ -2,12 +2,13 @@
 
 # called by dracut
 check() {
+    require_binaries "$systemdutildir"/systemd || return 1
     return 255
 }
 
-# called by dracut
+# called by dracut 
 depends() {
-    echo rootfs-block dm
+    echo systemd rootfs-block dm fs-lib
     return 0
 }
 
@@ -29,7 +30,7 @@ install() {
     # Include utilities required for cos-setup services,
     # probably a devoted cos-setup dracut module makes sense
     inst_multiple -o \
-        partprobe sync udevadm lsblk sgdisk parted mkfs.ext2 mkfs.ext3 mkfs.ext4 mkfs.vfat mkfs.fat mkfs.xfs blkid e2fsck resize2fs mount xfs_growfs umount
+        "$systemdutildir"/systemd-fsck partprobe sync udevadm lsblk sgdisk parted mkfs.ext2 mkfs.ext3 mkfs.ext4 mkfs.vfat mkfs.fat mkfs.xfs blkid e2fsck resize2fs mount xfs_growfs umount
     inst_hook cmdline 30 "${moddir}/parse-cos-cmdline.sh"
     inst_script "${moddir}/cos-generator.sh" \
         "${systemdutildir}/system-generators/dracut-cos-generator"
@@ -40,5 +41,7 @@ install() {
     mkdir -p "${initdir}/${systemdsystemunitdir}/initrd-fs.target.requires"
     ln_r "../cos-immutable-rootfs.service" \
         "${systemdsystemunitdir}/initrd-fs.target.requires/cos-immutable-rootfs.service"
+    ln_r "$systemdutildir"/systemd-fsck \
+        "/sbin/systemd-fsck"
     dracut_need_initqueue
-}
+}

packages/immutable-rootfs/build.yaml

@@ -7,7 +7,13 @@ steps:
 {{ if .Values.distribution }}
 {{if eq .Values.distribution "opensuse" }}
 # Mount /tmp as tmpfs by default as set by systemd itself
-- cp /usr/share/systemd/tmp.mount /etc/systemd/system
+- | 
+    /bin/bash -c " \
+    if [ -e /usr/share/systemd/tmp.mount ]; then \
+      cp /usr/share/systemd/tmp.mount /etc/systemd/system; \
+    else \
+      cp /usr/lib/systemd/system/tmp.mount /etc/systemd/system; \
+    fi "
 {{end}}
 {{end}}
 - cp -r 30cos-immutable-rootfs /usr/lib/dracut/modules.d

packages/immutable-rootfs/definition.yaml

@@ -1,6 +1,6 @@
 name: "immutable-rootfs"
 category: "system"
-version: 0.4-5
+version: "0.5"
 requires:
   - name: "cos-setup"
     category: "system"

packages/initrd/definition.yaml

@@ -1,4 +1,4 @@
 name: "dracut-initrd"
 category: "system"
-version: 0.2-8
+version: "0.3"
 description: "Dracut-based generated initrd"

tests/fallback/fallback_test.go

@@ -151,6 +151,43 @@ var _ = Describe("cOS booting fallback tests", func() {
 		})
 	})
 
+	Context("COS_PERSISTENT partition is corrupted", func() {
+		It("boots in active when the persistent partition is damaged, and can be repaired with fsck", func() {
+
+			// Just to make sure we can match against the same output of blkid later on
+			// and that the starting condition is the one we expect
+			Eventually(func() string {
+				out, _ := s.Command("sudo blkid")
+				return out
+			}, 1*time.Minute, 10*time.Second).Should(ContainSubstring(`LABEL="COS_PERSISTENT"`))
+
+			persistent, err := s.Command("blkid -L COS_PERSISTENT")
+			Expect(err).ToNot(HaveOccurred())
+
+			// This breaks the partition so it can be fixed with fsck
+			_, err = s.Command("dd if=/dev/zero count=1 bs=4096 seek=0 of=" + persistent)
+			Expect(err).ToNot(HaveOccurred())
+
+			Eventually(func() string {
+				out, _ := s.Command("sudo blkid")
+				return out
+			}, 5*time.Minute, 10*time.Second).ShouldNot(ContainSubstring(`LABEL="COS_PERSISTENT"`))
+
+			s.Reboot()
+			s.EventuallyConnects(700)
+
+			Expect(s.BootFrom()).To(Equal(sut.Active))
+
+			// We should see traces of fsck in the journal.
+			// Note, this is a bit ugly because the only messages
+			// we have from systemd-fsck is just failed attempts to run.
+			// But this is enough for us to assess if it actually kicked in.
+			out, err := s.Command("sudo journalctl")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(ContainSubstring("systemd-fsck"))
+		})
+	})
+
 	Context("GRUB cannot mount image", func() {
 		When("COS_ACTIVE image was corrupted", func() {
 			It("fallbacks by booting into passive", func() {