diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go index e9c42320f..0ef5c66a2 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject.go +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -99,7 +99,7 @@ func (h *globalMirrorPatchHandler) Mutate( if err != nil { return err } - files, generateErr := generateFiles( + files, commands, generateErr := generateFilesAndCommands( mirrorConfig, globalMirror) if generateErr != nil { @@ -115,6 +115,15 @@ func (h *globalMirrorPatchHandler) Mutate( files..., ) + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, + commands..., + ) + return nil }); err != nil { return err @@ -132,7 +141,7 @@ func (h *globalMirrorPatchHandler) Mutate( if err != nil { return err } - files, generateErr := generateFiles( + files, commands, generateErr := generateFilesAndCommands( mirrorConfig, globalMirror) if generateErr != nil { @@ -144,6 +153,13 @@ func (h *globalMirrorPatchHandler) Mutate( "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), ).Info("adding global registry mirror files to worker node kubeadm config template") obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to worker node kubeadm config template") + obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) + return nil }); err != nil { return err @@ -152,18 +168,29 @@ func (h *globalMirrorPatchHandler) Mutate( return nil } -func generateFiles( +func generateFilesAndCommands( mirrorConfig *mirrorConfig, globalMirror v1alpha1.GlobalImageRegistryMirror, -) ([]bootstrapv1.File, error) { - // Generate default registry mirror file +) ([]bootstrapv1.File, []string, error) { + // generate default registry mirror file files, err := generateGlobalRegistryMirrorFile(mirrorConfig) if err != nil { - return nil, err + return nil, nil, err } // generate CA certificate file for registry mirror mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror) files = append(files, mirrorCAFile...) + // generate Containerd registry config drop-in file + registryConfigDropIn := generateContainerdRegistryConfigDropInFile() + files = append(files, registryConfigDropIn...) + // generate Containerd apply patches script and command + var commands []string + applyPatchesFile, applyPatchesCommand, err := generateContainerdApplyPatchesScript() + if err != nil { + return nil, nil, err + } + files = append(files, applyPatchesFile...) + commands = append(commands, applyPatchesCommand) - return files, err + return files, commands, err } diff --git a/pkg/handlers/generic/mutation/mirrors/mirror.go b/pkg/handlers/generic/mutation/mirrors/mirror.go index 17a62b5ea..dc5cfc76a 100644 --- a/pkg/handlers/generic/mutation/mirrors/mirror.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror.go @@ -8,6 +8,7 @@ import ( "context" _ "embed" "fmt" + "path" "text/template" corev1 "k8s.io/api/core/v1" @@ -21,6 +22,11 @@ const ( mirrorCACertPathOnRemote = "/etc/certs/mirror.pem" defaultRegistryMirrorConfigPathOnRemote = "/etc/containerd/certs.d/_default/hosts.toml" secretKeyForMirrorCACert = "ca.crt" + + tomlMergeImage = "ghcr.io/mesosphere/toml-merge:v0.2.0" + containerdPatchesDirOnRemote = "/etc/containerd/cre.d" + containerdApplyPatchesScriptOnRemote = "/etc/containerd/apply-patches.sh" + containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote ) var ( @@ -30,6 +36,16 @@ var ( defaultRegistryMirrorPatchTemplate = template.Must( template.New("").Parse(string(defaultRegistryMirrorPatch)), ) + + //go:embed templates/containerd-registry-config-drop-in.toml + containerdRegistryConfigDropIn []byte + containerdRegistryConfigDropInFileOnRemote = path.Join( + containerdPatchesDirOnRemote, + "registry-config.toml", + ) + + //go:embed templates/containerd-apply-patches.sh.gotmpl + containerdApplyConfigPatchesScript []byte ) type mirrorConfig struct { @@ -124,10 +140,10 @@ func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, err } func generateMirrorCACertFile( - config *mirrorConfig, + mirror *mirrorConfig, globalMirror v1alpha1.GlobalImageRegistryMirror, ) []cabpkv1.File { - if config == nil || config.CACert == "" { + if mirror == nil || mirror.CACert == "" { return nil } return []cabpkv1.File{ @@ -143,3 +159,42 @@ func generateMirrorCACertFile( }, } } + +func generateContainerdRegistryConfigDropInFile() []cabpkv1.File { + return []cabpkv1.File{ + { + Path: containerdRegistryConfigDropInFileOnRemote, + Content: string(containerdRegistryConfigDropIn), + Permissions: "0600", + }, + } +} + +func generateContainerdApplyPatchesScript() ([]cabpkv1.File, string, error) { + t, err := template.New("").Parse(string(containerdApplyConfigPatchesScript)) + if err != nil { + return nil, "", fmt.Errorf("failed to parse go template: %w", err) + } + + templateInput := struct { + TOMLMergeImage string + PatchDir string + }{ + TOMLMergeImage: tomlMergeImage, + PatchDir: containerdPatchesDirOnRemote, + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + if err != nil { + return nil, "", fmt.Errorf("failed executing template: %w", err) + } + + return []cabpkv1.File{ + { + Path: containerdApplyPatchesScriptOnRemote, + Content: b.String(), + Permissions: "0700", + }, + }, containerdApplyPatchesScriptOnRemoteCommand, nil +} diff --git a/pkg/handlers/generic/mutation/mirrors/mirror_test.go b/pkg/handlers/generic/mutation/mirrors/mirror_test.go index 8a16bea61..0071df25d 100644 --- a/pkg/handlers/generic/mutation/mirrors/mirror_test.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror_test.go @@ -130,3 +130,58 @@ func Test_generateMirrorCACertFile(t *testing.T) { }) } } + +func Test_generateContainerdRegistryConfigDropInFile(t *testing.T) { + want := []cabpkv1.File{ + { + Path: "/etc/containerd/cre.d/registry-config.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +`, + }, + } + file := generateContainerdRegistryConfigDropInFile() + assert.Equal(t, want, file) +} + +func Test_generateContainerdApplyPatchesScript(t *testing.T) { + wantFile := []cabpkv1.File{ + { + Path: "/etc/containerd/apply-patches.sh", + Owner: "", + Permissions: "0700", + Encoding: "", + Append: false, + //nolint:lll // just a long string + Content: `#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +declare -r TOML_MERGE_IMAGE="ghcr.io/mesosphere/toml-merge:v0.2.0" + +if ! ctr --namespace k8s.io images check "name==${TOML_MERGE_IMAGE}" | grep "${TOML_MERGE_IMAGE}" >/dev/null; then + ctr --namespace k8s.io images pull "${TOML_MERGE_IMAGE}" +fi + +cleanup() { + ctr images unmount "${tmp_ctr_mount_dir}" || true +} + +trap 'cleanup' EXIT + +readonly tmp_ctr_mount_dir="$(mktemp -d)" + +ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}" +"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "/etc/containerd/cre.d/*.toml" /etc/containerd/config.toml +`, + }, + } + wantCmd := "/bin/bash /etc/containerd/apply-patches.sh" + file, cmd, _ := generateContainerdApplyPatchesScript() + assert.Equal(t, wantFile, file) + assert.Equal(t, wantCmd, cmd) +} diff --git a/pkg/handlers/generic/mutation/mirrors/templates/containerd-apply-patches.sh.gotmpl b/pkg/handlers/generic/mutation/mirrors/templates/containerd-apply-patches.sh.gotmpl new file mode 100644 index 000000000..f10c10c3f --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/templates/containerd-apply-patches.sh.gotmpl @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +declare -r TOML_MERGE_IMAGE="{{ .TOMLMergeImage }}" + +if ! ctr --namespace k8s.io images check "name==${TOML_MERGE_IMAGE}" | grep "${TOML_MERGE_IMAGE}" >/dev/null; then + ctr --namespace k8s.io images pull "${TOML_MERGE_IMAGE}" +fi + +cleanup() { + ctr images unmount "${tmp_ctr_mount_dir}" || true +} + +trap 'cleanup' EXIT + +readonly tmp_ctr_mount_dir="$(mktemp -d)" + +ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}" +"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "{{ .PatchDir }}/*.toml" /etc/containerd/config.toml diff --git a/pkg/handlers/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml b/pkg/handlers/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml new file mode 100644 index 000000000..e6adbd676 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml @@ -0,0 +1,2 @@ +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" diff --git a/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go index f4f40770e..117c37a3d 100644 --- a/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go @@ -69,6 +69,19 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/containerd/certs.d/_default/hosts.toml", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/cre.d/registry-config.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/apply-patches.sh", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/containerd/apply-patches.sh", ), }, }, @@ -101,6 +114,19 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/certs/mirror.pem", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/cre.d/registry-config.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/apply-patches.sh", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/containerd/apply-patches.sh", ), }, }, @@ -133,6 +159,19 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/containerd/certs.d/_default/hosts.toml", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/cre.d/registry-config.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/apply-patches.sh", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/containerd/apply-patches.sh", ), }, }, @@ -173,6 +212,19 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/certs/mirror.pem", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/cre.d/registry-config.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/apply-patches.sh", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/containerd/apply-patches.sh", ), }, },