Package e2fsprogs with ubuntu-image and select the appropriate conf at runtime

.github/workflows/build-and-test.yml

@@ -32,7 +32,7 @@ jobs:
         run: sudo apt update
 
       - name: install-test-dependencies
-        run: sudo apt install -y snapd germinate mtools debootstrap eatmydata fdisk gdisk qemu-system-aarch64 qemu-user-static ubuntu-dev-tools
+        run: sudo apt install -y snapd germinate e2fsprogs mtools debootstrap eatmydata fdisk gdisk qemu-system-aarch64 qemu-user-static ubuntu-dev-tools
 
       - name: short tests
         if: ${{ matrix.test-scenario == 'short' }}
@@ -58,6 +58,23 @@ jobs:
           path: .coverage/coverage-*.out
           retention-days: 1
 
+  mkfs-conf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: apt-update
+        run: sudo apt update
+
+      - name: install-test-dependencies
+        run: sudo apt install -y devscripts
+
+      - name: check mkfs conf updates
+        id: check-mkfs
+        shell: bash
+        run: ./tools/check-mkfs-confs.sh
+
   tics-report:
     runs-on: ubuntu-latest
     needs: test

CONTRIBUTING.md

@@ -125,6 +125,20 @@ git subtree pull --prefix tests/lib/external/snapd-testing-tools/ https://github
 ```
 
 
+## mkfs configuration
+
+ubuntu-image embeds in the snap different configurations to generate filesystems compatible with different series of ubuntu (and specifically different releases of `mkfs`). The snap building process should detect if one of these configurations is out of date and should fail. In this case, update the configurations by running the following from the project root directory:
+```
+# First install dependencies if needed
+sudo apt install ubuntu-dev-tools
+
+# Run the configuration updater
+./tools/collect-mkfs-confs.sh
+```
+
+Then check the configurations and the `./mkfs/db` file were updated. Commit the resulting changes.
+
+
 ## Release process
 
 ubuntu-image is released as a snap on the [Snap Store](https://snapcraft.io/ubuntu-image).

cmd/ubuntu-image/main_test.go

@@ -52,6 +52,10 @@ func (mockSM *mockedStateMachine) Teardown() error {
 func (mockSM *mockedStateMachine) SetCommonOpts(commonOpts *commands.CommonOpts, stateMachineOpts *commands.StateMachineOpts) {
 }
 
+func (mockSM *mockedStateMachine) SetSeries() error {
+	return nil
+}
+
 // TestValidCommands tests that certain valid commands are parsed correctly
 func TestValidCommands(t *testing.T) {
 	t.Parallel()

go.mod

@@ -35,7 +35,7 @@ require (
 
 require (
 	github.com/snapcore/secboot v0.0.0-20240411101434-f3ad7c92552a // indirect
-	github.com/snapcore/snapd v0.0.0-20240429124543-2ea10e81aa88
+	github.com/snapcore/snapd v0.0.0-20240618105125-0d60393deebb
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -56,7 +56,7 @@ require (
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
-	github.com/canonical/go-efilib v0.3.1-0.20220815143333-7e5151412e93 // indirect
+	github.com/canonical/go-efilib v0.4.0 // indirect
 	github.com/canonical/go-sp800.108-kdf v0.0.0-20210314145419-a3359f2d21b9 // indirect
 	github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 // indirect
 	github.com/canonical/go-tpm2 v0.0.0-20210827151749-f80ff5afff61 // indirect

go.sum

@@ -14,8 +14,8 @@ github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xW
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
-github.com/canonical/go-efilib v0.3.1-0.20220815143333-7e5151412e93 h1:F0bRDzPy/j2IX/iIWqCEA23S1nal+f7A+/vLyj6Ye+4=
-github.com/canonical/go-efilib v0.3.1-0.20220815143333-7e5151412e93/go.mod h1:9b2PNAuPcZsB76x75/uwH99D8CyH/A2y4rq1/+bvplg=
+github.com/canonical/go-efilib v0.4.0 h1:2ee5pvhIZ+g1EO4HxFE/owBgs5Up2g7dw1+Ls9/fiSs=
+github.com/canonical/go-efilib v0.4.0/go.mod h1:9b2PNAuPcZsB76x75/uwH99D8CyH/A2y4rq1/+bvplg=
 github.com/canonical/go-sp800.108-kdf v0.0.0-20210314145419-a3359f2d21b9 h1:USzKjrfWo/ESzozv2i3OMM7XDgxrZRvaHFrKkIKRtwU=
 github.com/canonical/go-sp800.108-kdf v0.0.0-20210314145419-a3359f2d21b9/go.mod h1:Zrs3YjJr+w51u0R/dyLh/oWt/EcBVdLPCVFYC4daW5s=
 github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw=
@@ -121,8 +121,8 @@ github.com/snapcore/go-gettext v0.0.0-20230721153050-9082cdc2db05/go.mod h1:1ueM
 github.com/snapcore/secboot v0.0.0-20240411101434-f3ad7c92552a h1:yzzVi0yUosDYkjSQqGZNVtaVi+6yNFLiF0erKHlBbdo=
 github.com/snapcore/secboot v0.0.0-20240411101434-f3ad7c92552a/go.mod h1:72paVOkm4sJugXt+v9ItmnjXgO921D8xqsbH2OekouY=
 github.com/snapcore/snapd v0.0.0-20201005140838-501d14ac146e/go.mod h1:3xrn7QDDKymcE5VO2rgWEQ5ZAUGb9htfwlXnoel6Io8=
-github.com/snapcore/snapd v0.0.0-20240429124543-2ea10e81aa88 h1:RecfXpnAX89weUaNZE/NqH0A08qYWoNd/bA8Pn4yAKc=
-github.com/snapcore/snapd v0.0.0-20240429124543-2ea10e81aa88/go.mod h1:QNsQNTz6P2yqZ22QTU5Jt1FUIGGLlcUuNURvo2tWYOo=
+github.com/snapcore/snapd v0.0.0-20240618105125-0d60393deebb h1:bnzbC+m9mF7ZM6JUvU7GmS/N6y+m9f2Agoo91iQhS78=
+github.com/snapcore/snapd v0.0.0-20240618105125-0d60393deebb/go.mod h1:RfaJNlcyEP2RfVuyAX/3x+CJPECCAlZn+36Yily9FzY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

internal/statemachine/classic.go

@@ -66,6 +66,10 @@ func (classicStateMachine *ClassicStateMachine) Setup() error {
 		return err
 	}
 
+	if err := classicStateMachine.SetSeries(); err != nil {
+		return err
+	}
+
 	classicStateMachine.displayStates()
 
 	if classicStateMachine.commonFlags.DryRun {
@@ -79,6 +83,11 @@ func (classicStateMachine *ClassicStateMachine) Setup() error {
 	return classicStateMachine.determineOutputDirectory()
 }
 
+func (classicStateMachine *ClassicStateMachine) SetSeries() error {
+	classicStateMachine.series = classicStateMachine.ImageDef.Series
+	return nil
+}
+
 // parseImageDefinition parses the provided yaml file and ensures it is valid
 func (stateMachine *StateMachine) parseImageDefinition() error {
 	classicStateMachine := stateMachine.parent.(*ClassicStateMachine)

internal/statemachine/helper.go

@@ -40,6 +40,12 @@ const (
 var runCmd = helper.RunCmd
 var blockSize string = "1"
 
+var (
+	MKE2FS_CONFIG_ENV  = "MKE2FS_CONFIG"
+	MKE2FS_CONFIG_FILE = "mke2fs.conf"
+	MKE2FS_BASE_PATH   = "/etc/ubuntu-image/mkfs"
+)
+
 // validateInput ensures that command line flags for the state machine are valid. These
 // flags are applicable to all image types
 func (stateMachine *StateMachine) validateInput() error {
@@ -237,7 +243,7 @@ func (stateMachine *StateMachine) prepareAndCreateFS(volume *gadget.Volume, stru
 		return err
 	}
 
-	return makeFS(structure, contentRoot, partImg, stateMachine.SectorSize)
+	return makeFS(structure, contentRoot, partImg, stateMachine.SectorSize, stateMachine.series)
 }
 
 // prepareDiskImg prepares a raw image
@@ -264,12 +270,18 @@ func prepareDiskImg(structure gadget.VolumeStructure, partImg string, blockSize
 }
 
 // makeFS actually creates the filesystem for the given structure
-func makeFS(structure gadget.VolumeStructure, contentRoot string, partImg string, sectorSize quantity.Size) error {
+func makeFS(structure gadget.VolumeStructure, contentRoot string, partImg string, sectorSize quantity.Size, series string) error {
 	hasC, err := hasContent(structure, contentRoot)
 	if err != nil {
 		return err
 	}
 
+	// select the mkfs.ext4 conf to use
+	err = setMk2fsConf(series)
+	if err != nil {
+		return fmt.Errorf("Error preparing env for mkfs: %s", err.Error())
+	}
+
 	if hasC {
 		err := mkfsMakeWithContent(structure.Filesystem, partImg, structure.Label,
 			contentRoot, structure.Size, sectorSize)
@@ -318,6 +330,19 @@ func fixDiskIDOnMBR(imgName string) error {
 	return nil
 }
 
+// The MKE2FS_BASE_PATH folder is setup to handle codename and release number as a series.
+func setMk2fsConf(series string) error {
+	mk2fsConfPath := strings.Join([]string{osGetenv("SNAP"), MKE2FS_BASE_PATH, series, MKE2FS_CONFIG_FILE}, "/")
+
+	_, err := os.Stat(mk2fsConfPath)
+	if err != nil {
+		fmt.Printf("WARNING: No mkfs configuration found for this series: %s. Will fallback on the default one.\n", series)
+		return nil
+	}
+
+	return osSetenv(MKE2FS_CONFIG_ENV, mk2fsConfPath)
+}
+
 // handleSecureBoot handles a special case where files need to be moved from /boot/ to
 // /EFI/ubuntu/ so that SecureBoot can still be used
 func (stateMachine *StateMachine) handleSecureBoot(volume *gadget.Volume, targetDir string) error {

internal/statemachine/helper_test.go

@@ -232,7 +232,9 @@ func TestFailedHandleLkBootloader(t *testing.T) {
 // functions and setting invalid bs= arguments in dd
 func TestFailedCopyStructureContent(t *testing.T) {
 	asserter := helper.Asserter{T: t}
-	var stateMachine StateMachine
+	stateMachine := StateMachine{
+		series: "mantic",
+	}
 	stateMachine.commonFlags, stateMachine.stateMachineFlags = helper.InitCommonOpts()
 	stateMachine.YamlFilePath = filepath.Join("testdata", "gadget_tree",
 		"meta", "gadget.yaml")
@@ -295,6 +297,26 @@ func TestFailedCopyStructureContent(t *testing.T) {
 	asserter.AssertErrContains(err, "Error listing contents of volume")
 	osReadDir = os.ReadDir
 
+	// Set invalid value in MKE2FS_CONFIG_ENV
+	OLD_MKE2FS_CONFIG_ENV := MKE2FS_CONFIG_ENV
+	MKE2FS_CONFIG_ENV = "test="
+
+	err = os.Setenv("SNAP", "testdata/mkfs")
+	asserter.AssertErrNil(err, true)
+
+	OLD_MKE2FS_BASE_PATH := MKE2FS_BASE_PATH
+	MKE2FS_BASE_PATH = "base_path_test"
+
+	t.Cleanup(func() {
+		MKE2FS_BASE_PATH = OLD_MKE2FS_BASE_PATH
+	})
+
+	err = stateMachine.copyStructureContent(volume, rootfsStruct, 0, "",
+		filepath.Join("/tmp", uuid.NewString()+".img"))
+	asserter.AssertErrContains(err, "Error preparing env for mkfs")
+	MKE2FS_CONFIG_ENV = OLD_MKE2FS_CONFIG_ENV
+	MKE2FS_BASE_PATH = OLD_MKE2FS_BASE_PATH
+
 	// mock gadget.MkfsWithContent
 	mkfsMakeWithContent = mockMkfsWithContent
 	defer func() {
@@ -1273,3 +1295,106 @@ func TestStateMachine_setConfDefDir(t *testing.T) {
 		})
 	}
 }
+
+type mockEnvHolder struct {
+	env        map[string]string
+	mockGetenv bool // mock or use the real os.Getenv implementation
+	mockSetenv bool // mock or use the real os.Setenv implementation
+}
+
+func (m *mockEnvHolder) Getenv(key string) string {
+	if m.mockGetenv {
+		return m.env[key]
+	}
+	return os.Getenv(key)
+}
+
+func (m *mockEnvHolder) Setenv(key, value string) error {
+	if m.mockSetenv {
+		m.env[key] = value
+		return nil
+	}
+
+	return os.Setenv(key, value)
+}
+
+func TestStateMachine_setMk2fsConf(t *testing.T) {
+	type fields struct {
+		series string
+	}
+	tests := []struct {
+		name         string
+		fields       fields
+		mkfsBasePath string
+		envHolder    *mockEnvHolder
+		want         string
+	}{
+		{
+			name: "set env with sucess",
+			fields: fields{
+				series: "mantic",
+			},
+			mkfsBasePath: "base_path_test",
+			envHolder: &mockEnvHolder{
+				env:        map[string]string{"SNAP": "testdata/mkfs"},
+				mockGetenv: true,
+				mockSetenv: true,
+			},
+			want: "testdata/mkfs/base_path_test/mantic/mke2fs.conf",
+		},
+		{
+			name: "set env with sucess with current const values",
+			fields: fields{
+				series: "mantic",
+			},
+			mkfsBasePath: "base_path_test",
+			envHolder: &mockEnvHolder{
+				env:        map[string]string{"SNAP": "testdata/mkfs"},
+				mockGetenv: true,
+				mockSetenv: false,
+			},
+			want: "testdata/mkfs/base_path_test/mantic/mke2fs.conf",
+		},
+		{
+			name: "fail to set env to inexistent file",
+			fields: fields{
+				series: "mantic",
+			},
+			envHolder: &mockEnvHolder{
+				env:        map[string]string{"SNAP": "testdata/mkfs"},
+				mockGetenv: true,
+				mockSetenv: true,
+			},
+			mkfsBasePath: "inexistent",
+			want:         "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			asserter := &helper.Asserter{T: t}
+
+			osGetenv = tt.envHolder.Getenv
+			osSetenv = tt.envHolder.Setenv
+			OLD_MKE2FS_BASE_PATH := MKE2FS_BASE_PATH
+			MKE2FS_BASE_PATH = tt.mkfsBasePath
+
+			t.Cleanup(func() {
+				osGetenv = os.Getenv
+				osSetenv = os.Setenv
+				MKE2FS_BASE_PATH = OLD_MKE2FS_BASE_PATH
+				os.Unsetenv(MKE2FS_CONFIG_ENV)
+			})
+
+			err := setMk2fsConf(tt.fields.series)
+			asserter.AssertErrNil(err, true)
+
+			var got string
+			if tt.envHolder.mockSetenv {
+				got = tt.envHolder.env[MKE2FS_CONFIG_ENV]
+			} else {
+				got = os.Getenv(MKE2FS_CONFIG_ENV)
+			}
+			asserter.AssertEqual(tt.want, got)
+		})
+	}
+}

internal/statemachine/pack.go

@@ -57,3 +57,8 @@ func (packStateMachine *PackStateMachine) Setup() error {
 
 	return packStateMachine.makeTemporaryDirectories()
 }
+
+// Placeholder method to satisfy the interface. This is not used when packing.
+func (packStateMachine *PackStateMachine) SetSeries() error {
+	return nil
+}