From 1b1e6ccab0a740a34ef7ec4c2524e9658f60961d Mon Sep 17 00:00:00 2001 From: Ahmed AbouZaid Date: Sun, 9 Oct 2022 15:16:47 +0200 Subject: [PATCH 1/4] Test transformers krm exec function --- api/krusty/fnplugin_test.go | 126 +++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 1 deletion(-) diff --git a/api/krusty/fnplugin_test.go b/api/krusty/fnplugin_test.go index a4be95d557..81f6f3c824 100644 --- a/api/krusty/fnplugin_test.go +++ b/api/krusty/fnplugin_test.go @@ -41,6 +41,12 @@ spec: EOF ` +const krmEchoDotSh = `#!/bin/bash + +resourceList=$(cat) +echo "$resourceList" +` + func TestFnExecGenerator(t *testing.T) { fSys := filesys.MakeFsOnDisk() @@ -126,7 +132,7 @@ spec: assert.NoError(t, fSys.RemoveAll(tmpDir.String())) } -func TestFnExecGeneratorWithOverlay(t *testing.T) { +func TestFnExecGeneratorInOverlay(t *testing.T) { fSys := filesys.MakeFsOnDisk() th := kusttest_test.MakeHarnessWithFs(t, fSys) @@ -217,6 +223,124 @@ spec: assert.NoError(t, fSys.RemoveAll(tmpDir.String())) } +func TestFnExecTransformer(t *testing.T) { + fSys := filesys.MakeFsOnDisk() + + th := kusttest_test.MakeHarnessWithFs(t, fSys) + o := th.MakeOptionsPluginsEnabled() + o.PluginConfig.FnpLoadingOptions.EnableExec = true + + tmpDir, err := filesys.NewTmpConfirmedDir() + assert.NoError(t, err) + base := filepath.Join(tmpDir.String(), "base") + assert.NoError(t, fSys.Mkdir(base)) + th.WriteK(base, ` +resources: +- secret.yaml +transformers: +- krm-echo.yaml +`) + th.WriteF(filepath.Join(base, "secret.yaml"), + ` +apiVersion: v1 +kind: Secret +metadata: + name: dummy +type: Opaque +stringData: + foo: bar +`) + th.WriteF(filepath.Join(base, "krmEcho.sh"), krmEchoDotSh) + + assert.NoError(t, os.Chmod(filepath.Join(base, "krmEcho.sh"), 0777)) + th.WriteF(filepath.Join(base, "krm-echo.yaml"), ` +apiVersion: examples.config.kubernetes.io/v1beta1 +kind: MyPlugin +metadata: + name: notImportantHere + annotations: + config.kubernetes.io/function: | + exec: + path: ./krmEcho.sh +`) + + m := th.Run(base, o) + assert.NoError(t, err) + yml, err := m.AsYaml() + assert.NoError(t, err) + assert.Equal(t, `apiVersion: v1 +kind: Secret +metadata: + name: dummy +stringData: + foo: bar +type: Opaque +`, string(yml)) + assert.NoError(t, fSys.RemoveAll(tmpDir.String())) +} + +func TestFnExecTransformerInOverlay(t *testing.T) { + fSys := filesys.MakeFsOnDisk() + + th := kusttest_test.MakeHarnessWithFs(t, fSys) + o := th.MakeOptionsPluginsEnabled() + o.PluginConfig.FnpLoadingOptions.EnableExec = true + + tmpDir, err := filesys.NewTmpConfirmedDir() + assert.NoError(t, err) + base := filepath.Join(tmpDir.String(), "base") + prod := filepath.Join(tmpDir.String(), "prod") + assert.NoError(t, fSys.Mkdir(base)) + assert.NoError(t, fSys.Mkdir(prod)) + th.WriteK(base, ` +resources: +- secret.yaml +`) + th.WriteK(prod, ` +resources: +- ../base +transformers: +- krm-echo.yaml +`) + th.WriteF(filepath.Join(base, "secret.yaml"), + ` +apiVersion: v1 +kind: Secret +metadata: + name: dummy +type: Opaque +stringData: + foo: bar +`) + th.WriteF(filepath.Join(prod, "krmEcho.sh"), krmEchoDotSh) + + assert.NoError(t, os.Chmod(filepath.Join(prod, "krmEcho.sh"), 0777)) + th.WriteF(filepath.Join(prod, "krm-echo.yaml"), ` +apiVersion: examples.config.kubernetes.io/v1beta1 +kind: MyPlugin +metadata: + name: notImportantHere + annotations: + config.kubernetes.io/function: | + exec: + path: ./krmEcho.sh +`) + + m := th.Run(prod, o) + assert.NoError(t, err) + yml, err := m.AsYaml() + assert.NoError(t, err) + assert.Equal(t, `apiVersion: v1 +kind: Secret +metadata: + name: dummy +stringData: + foo: bar +type: Opaque +`, string(yml)) + assert.NoError(t, fSys.RemoveAll(tmpDir.String())) +} + func skipIfNoDocker(t *testing.T) { t.Helper() if _, err := exec.LookPath("docker"); err != nil { From d29febecb71aebc7b6db89e4e6071ed0ccdfdc7f Mon Sep 17 00:00:00 2001 From: Ahmed AbouZaid Date: Sun, 9 Oct 2022 15:22:42 +0200 Subject: [PATCH 2/4] Fix krm exec function working dir --- api/internal/plugins/loader/loader.go | 14 ++++++++++++-- api/internal/plugins/loader/loader_test.go | 14 ++++++++++++++ api/internal/target/kusttarget.go | 4 +++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/api/internal/plugins/loader/loader.go b/api/internal/plugins/loader/loader.go index 26441ed51c..36d929d7d3 100644 --- a/api/internal/plugins/loader/loader.go +++ b/api/internal/plugins/loader/loader.go @@ -47,8 +47,18 @@ func (l *Loader) Config() *types.PluginConfig { return l.pc } -// SetWorkDir sets the working directory for this loader's plugins -func (l *Loader) SetWorkDir(wd string) { +// DeepCopyPluginConfig makes a full copy the actual values of PluginConfig. +func (l *Loader) DeepCopyPluginConfig() { + l.pc = &types.PluginConfig{ + PluginRestrictions: l.pc.PluginRestrictions, + BpLoadingOptions: l.pc.BpLoadingOptions, + FnpLoadingOptions: l.pc.FnpLoadingOptions, + HelmConfig: l.pc.HelmConfig, + } +} + +// SetPluginConfigWorkingDir sets the working directory for the loader's plugins. +func (l *Loader) SetPluginConfigWorkingDir(wd string) { l.pc.FnpLoadingOptions.WorkingDir = wd } diff --git a/api/internal/plugins/loader/loader_test.go b/api/internal/plugins/loader/loader_test.go index c5e30b808b..ba40762b82 100644 --- a/api/internal/plugins/loader/loader_test.go +++ b/api/internal/plugins/loader/loader_test.go @@ -78,3 +78,17 @@ func TestLoader(t *testing.T) { } } } + +func TestLoaderSetPluginConfigWorkingDir(t *testing.T) { + p := provider.NewDefaultDepProvider() + rmF := resmap.NewFactory(p.GetResourceFactory()) + fsys := filesys.MakeFsInMemory() + c := types.EnabledPluginConfig(types.BploLoadFromFileSys) + pLdr := NewLoader(c, rmF, fsys) + pLdrCopy := *pLdr + pLdrCopy.DeepCopyPluginConfig() + pLdrCopy.SetPluginConfigWorkingDir("/tmp/dummy") + if pLdrCopy.Config().FnpLoadingOptions.WorkingDir != "/tmp/dummy" { + t.Fatal("plugin working dir is not set correctly") + } +} diff --git a/api/internal/target/kusttarget.go b/api/internal/target/kusttarget.go index 4bbf7e98a6..17318c3960 100644 --- a/api/internal/target/kusttarget.go +++ b/api/internal/target/kusttarget.go @@ -45,7 +45,9 @@ func NewKustTarget( rFactory *resmap.Factory, pLdr *loader.Loader) *KustTarget { pLdrCopy := *pLdr - pLdrCopy.SetWorkDir(ldr.Root()) + pLdrCopy.DeepCopyPluginConfig() + pLdrCopy.SetPluginConfigWorkingDir(ldr.Root()) + return &KustTarget{ ldr: ldr, validator: validator, From ea21b37d67f27c22f2d8fce76794bbe4b8a21d4d Mon Sep 17 00:00:00 2001 From: Ahmed AbouZaid Date: Sat, 15 Oct 2022 16:45:50 +0200 Subject: [PATCH 3/4] proposal v2 --- api/internal/plugins/loader/loader.go | 20 +++++++++----------- api/internal/plugins/loader/loader_test.go | 14 +++++++------- api/internal/target/kusttarget.go | 6 +----- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/api/internal/plugins/loader/loader.go b/api/internal/plugins/loader/loader.go index 36d929d7d3..43b11d6ffb 100644 --- a/api/internal/plugins/loader/loader.go +++ b/api/internal/plugins/loader/loader.go @@ -42,24 +42,22 @@ func NewLoader( return &Loader{pc: pc, rf: rf, fs: fs} } -// Config provides the global (not plugin specific) PluginConfig data. -func (l *Loader) Config() *types.PluginConfig { - return l.pc -} - -// DeepCopyPluginConfig makes a full copy the actual values of PluginConfig. -func (l *Loader) DeepCopyPluginConfig() { - l.pc = &types.PluginConfig{ +// LoaderWithWorkingDir returns loader after setting its working directory. +// NOTE: This is not really a new loader since some of the Loader struct fields are pointers. +func (l *Loader) LoaderWithWorkingDir(wd string) *Loader { + lpc := &types.PluginConfig{ PluginRestrictions: l.pc.PluginRestrictions, BpLoadingOptions: l.pc.BpLoadingOptions, FnpLoadingOptions: l.pc.FnpLoadingOptions, HelmConfig: l.pc.HelmConfig, } + lpc.FnpLoadingOptions.WorkingDir = wd + return &Loader{pc: lpc, rf: l.rf, fs: l.fs} } -// SetPluginConfigWorkingDir sets the working directory for the loader's plugins. -func (l *Loader) SetPluginConfigWorkingDir(wd string) { - l.pc.FnpLoadingOptions.WorkingDir = wd +// Config provides the global (not plugin specific) PluginConfig data. +func (l *Loader) Config() *types.PluginConfig { + return l.pc } func (l *Loader) LoadGenerators( diff --git a/api/internal/plugins/loader/loader_test.go b/api/internal/plugins/loader/loader_test.go index ba40762b82..53992db52b 100644 --- a/api/internal/plugins/loader/loader_test.go +++ b/api/internal/plugins/loader/loader_test.go @@ -6,6 +6,7 @@ package loader_test import ( "testing" + "github.com/stretchr/testify/require" . "sigs.k8s.io/kustomize/api/internal/plugins/loader" "sigs.k8s.io/kustomize/api/loader" "sigs.k8s.io/kustomize/api/provider" @@ -79,16 +80,15 @@ func TestLoader(t *testing.T) { } } -func TestLoaderSetPluginConfigWorkingDir(t *testing.T) { +func TestLoaderWithWorkingDir(t *testing.T) { p := provider.NewDefaultDepProvider() rmF := resmap.NewFactory(p.GetResourceFactory()) fsys := filesys.MakeFsInMemory() c := types.EnabledPluginConfig(types.BploLoadFromFileSys) pLdr := NewLoader(c, rmF, fsys) - pLdrCopy := *pLdr - pLdrCopy.DeepCopyPluginConfig() - pLdrCopy.SetPluginConfigWorkingDir("/tmp/dummy") - if pLdrCopy.Config().FnpLoadingOptions.WorkingDir != "/tmp/dummy" { - t.Fatal("plugin working dir is not set correctly") - } + npLdr := pLdr.LoaderWithWorkingDir("/tmp/dummy") + require.Equal(t, + "/tmp/dummy", + npLdr.Config().FnpLoadingOptions.WorkingDir, + "plugin working dir is not set correctly") } diff --git a/api/internal/target/kusttarget.go b/api/internal/target/kusttarget.go index 17318c3960..d5d7c1474e 100644 --- a/api/internal/target/kusttarget.go +++ b/api/internal/target/kusttarget.go @@ -44,15 +44,11 @@ func NewKustTarget( validator ifc.Validator, rFactory *resmap.Factory, pLdr *loader.Loader) *KustTarget { - pLdrCopy := *pLdr - pLdrCopy.DeepCopyPluginConfig() - pLdrCopy.SetPluginConfigWorkingDir(ldr.Root()) - return &KustTarget{ ldr: ldr, validator: validator, rFactory: rFactory, - pLdr: &pLdrCopy, + pLdr: pLdr.LoaderWithWorkingDir(ldr.Root()), } } From 3e447da6ef302a24d24b83794f7ccff0c75081af Mon Sep 17 00:00:00 2001 From: Ahmed AbouZaid Date: Sat, 22 Oct 2022 02:50:28 +0200 Subject: [PATCH 4/4] proposal v2.1 --- api/internal/plugins/loader/loader_test.go | 6 +- api/krusty/fnplugin_test.go | 197 ++++++++++++++++++--- 2 files changed, 181 insertions(+), 22 deletions(-) diff --git a/api/internal/plugins/loader/loader_test.go b/api/internal/plugins/loader/loader_test.go index 53992db52b..23cf55cb95 100644 --- a/api/internal/plugins/loader/loader_test.go +++ b/api/internal/plugins/loader/loader_test.go @@ -87,8 +87,12 @@ func TestLoaderWithWorkingDir(t *testing.T) { c := types.EnabledPluginConfig(types.BploLoadFromFileSys) pLdr := NewLoader(c, rmF, fsys) npLdr := pLdr.LoaderWithWorkingDir("/tmp/dummy") + require.Equal(t, + "", + pLdr.Config().FnpLoadingOptions.WorkingDir, + "the plugin working dir should not change") require.Equal(t, "/tmp/dummy", npLdr.Config().FnpLoadingOptions.WorkingDir, - "plugin working dir is not set correctly") + "the plugin working dir is not updated") } diff --git a/api/krusty/fnplugin_test.go b/api/krusty/fnplugin_test.go index 81f6f3c824..40f63b6427 100644 --- a/api/krusty/fnplugin_test.go +++ b/api/krusty/fnplugin_test.go @@ -41,13 +41,19 @@ spec: EOF ` -const krmEchoDotSh = `#!/bin/bash - -resourceList=$(cat) -echo "$resourceList" +const krmTransformerDotSh = `#!/bin/bash +cat << EOF +apiVersion: v1 +kind: Secret +metadata: + name: dummyTransformed +stringData: + foo: bar +type: Opaque +EOF ` -func TestFnExecGenerator(t *testing.T) { +func TestFnExecGeneratorInBase(t *testing.T) { fSys := filesys.MakeFsOnDisk() th := kusttest_test.MakeHarnessWithFs(t, fSys) @@ -93,7 +99,6 @@ spec: `) m := th.Run(tmpDir.String(), o) - assert.NoError(t, err) yml, err := m.AsYaml() assert.NoError(t, err) assert.Equal(t, `apiVersion: v1 @@ -132,7 +137,7 @@ spec: assert.NoError(t, fSys.RemoveAll(tmpDir.String())) } -func TestFnExecGeneratorInOverlay(t *testing.T) { +func TestFnExecGeneratorInBaseWithOverlay(t *testing.T) { fSys := filesys.MakeFsOnDisk() th := kusttest_test.MakeHarnessWithFs(t, fSys) @@ -223,7 +228,98 @@ spec: assert.NoError(t, fSys.RemoveAll(tmpDir.String())) } -func TestFnExecTransformer(t *testing.T) { +func TestFnExecGeneratorInOverlay(t *testing.T) { + fSys := filesys.MakeFsOnDisk() + + th := kusttest_test.MakeHarnessWithFs(t, fSys) + o := th.MakeOptionsPluginsEnabled() + o.PluginConfig.FnpLoadingOptions.EnableExec = true + + tmpDir, err := filesys.NewTmpConfirmedDir() + assert.NoError(t, err) + base := filepath.Join(tmpDir.String(), "base") + prod := filepath.Join(tmpDir.String(), "prod") + assert.NoError(t, fSys.Mkdir(base)) + assert.NoError(t, fSys.Mkdir(prod)) + th.WriteK(base, ` +resources: +- short_secret.yaml +`) + th.WriteK(prod, ` +resources: +- ../base +generators: +- gener.yaml +`) + th.WriteF(filepath.Join(base, "short_secret.yaml"), + ` +apiVersion: v1 +kind: Secret +metadata: + labels: + airshipit.org/ephemeral-user-data: "true" + name: node1-bmc-secret +type: Opaque +stringData: + userData: | + bootcmd: + - mkdir /mnt/vda +`) + th.WriteF(filepath.Join(prod, "generateDeployment.sh"), generateDeploymentDotSh) + + assert.NoError(t, os.Chmod(filepath.Join(prod, "generateDeployment.sh"), 0777)) + th.WriteF(filepath.Join(prod, "gener.yaml"), ` +kind: executable +metadata: + name: demo + annotations: + config.kubernetes.io/function: | + exec: + path: ./generateDeployment.sh +spec: +`) + + m := th.Run(prod, o) + assert.NoError(t, err) + yml, err := m.AsYaml() + assert.NoError(t, err) + assert.Equal(t, `apiVersion: v1 +kind: Secret +metadata: + labels: + airshipit.org/ephemeral-user-data: "true" + name: node1-bmc-secret +stringData: + userData: | + bootcmd: + - mkdir /mnt/vda +type: Opaque +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + tshirt-size: small + labels: + app: nginx + name: nginx +spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - image: nginx + name: nginx +`, string(yml)) + assert.NoError(t, fSys.RemoveAll(tmpDir.String())) +} + +func TestFnExecTransformerInBase(t *testing.T) { fSys := filesys.MakeFsOnDisk() th := kusttest_test.MakeHarnessWithFs(t, fSys) @@ -238,7 +334,7 @@ func TestFnExecTransformer(t *testing.T) { resources: - secret.yaml transformers: -- krm-echo.yaml +- krm-transformer.yaml `) th.WriteF(filepath.Join(base, "secret.yaml"), ` @@ -250,10 +346,10 @@ type: Opaque stringData: foo: bar `) - th.WriteF(filepath.Join(base, "krmEcho.sh"), krmEchoDotSh) + th.WriteF(filepath.Join(base, "krmTransformer.sh"), krmTransformerDotSh) - assert.NoError(t, os.Chmod(filepath.Join(base, "krmEcho.sh"), 0777)) - th.WriteF(filepath.Join(base, "krm-echo.yaml"), ` + assert.NoError(t, os.Chmod(filepath.Join(base, "krmTransformer.sh"), 0777)) + th.WriteF(filepath.Join(base, "krm-transformer.yaml"), ` apiVersion: examples.config.kubernetes.io/v1beta1 kind: MyPlugin metadata: @@ -261,17 +357,77 @@ metadata: annotations: config.kubernetes.io/function: | exec: - path: ./krmEcho.sh + path: ./krmTransformer.sh `) m := th.Run(base, o) - assert.NoError(t, err) yml, err := m.AsYaml() assert.NoError(t, err) assert.Equal(t, `apiVersion: v1 kind: Secret +metadata: + name: dummyTransformed +stringData: + foo: bar +type: Opaque +`, string(yml)) + assert.NoError(t, fSys.RemoveAll(tmpDir.String())) +} + +func TestFnExecTransformerInBaseWithOverlay(t *testing.T) { + fSys := filesys.MakeFsOnDisk() + + th := kusttest_test.MakeHarnessWithFs(t, fSys) + o := th.MakeOptionsPluginsEnabled() + o.PluginConfig.FnpLoadingOptions.EnableExec = true + + tmpDir, err := filesys.NewTmpConfirmedDir() + assert.NoError(t, err) + base := filepath.Join(tmpDir.String(), "base") + prod := filepath.Join(tmpDir.String(), "prod") + assert.NoError(t, fSys.Mkdir(base)) + assert.NoError(t, fSys.Mkdir(prod)) + th.WriteK(base, ` +resources: +- secret.yaml +transformers: +- krm-transformer.yaml +`) + th.WriteK(prod, ` +resources: +- ../base +`) + th.WriteF(filepath.Join(base, "secret.yaml"), + ` +apiVersion: v1 +kind: Secret metadata: name: dummy +type: Opaque +stringData: + foo: bar +`) + th.WriteF(filepath.Join(base, "krmTransformer.sh"), krmTransformerDotSh) + + assert.NoError(t, os.Chmod(filepath.Join(base, "krmTransformer.sh"), 0777)) + th.WriteF(filepath.Join(base, "krm-transformer.yaml"), ` +apiVersion: examples.config.kubernetes.io/v1beta1 +kind: MyPlugin +metadata: + name: notImportantHere + annotations: + config.kubernetes.io/function: | + exec: + path: ./krmTransformer.sh +`) + + m := th.Run(prod, o) + yml, err := m.AsYaml() + assert.NoError(t, err) + assert.Equal(t, `apiVersion: v1 +kind: Secret +metadata: + name: dummyTransformed stringData: foo: bar type: Opaque @@ -300,7 +456,7 @@ resources: resources: - ../base transformers: -- krm-echo.yaml +- krm-transformer.yaml `) th.WriteF(filepath.Join(base, "secret.yaml"), ` @@ -312,10 +468,10 @@ type: Opaque stringData: foo: bar `) - th.WriteF(filepath.Join(prod, "krmEcho.sh"), krmEchoDotSh) + th.WriteF(filepath.Join(prod, "krmTransformer.sh"), krmTransformerDotSh) - assert.NoError(t, os.Chmod(filepath.Join(prod, "krmEcho.sh"), 0777)) - th.WriteF(filepath.Join(prod, "krm-echo.yaml"), ` + assert.NoError(t, os.Chmod(filepath.Join(prod, "krmTransformer.sh"), 0777)) + th.WriteF(filepath.Join(prod, "krm-transformer.yaml"), ` apiVersion: examples.config.kubernetes.io/v1beta1 kind: MyPlugin metadata: @@ -323,17 +479,16 @@ metadata: annotations: config.kubernetes.io/function: | exec: - path: ./krmEcho.sh + path: ./krmTransformer.sh `) m := th.Run(prod, o) - assert.NoError(t, err) yml, err := m.AsYaml() assert.NoError(t, err) assert.Equal(t, `apiVersion: v1 kind: Secret metadata: - name: dummy + name: dummyTransformed stringData: foo: bar type: Opaque