From f121e747444657e32bd69f8038b509087af6c81c Mon Sep 17 00:00:00 2001 From: Natasha Sarkar Date: Thu, 3 Jun 2021 14:54:41 -0700 Subject: [PATCH] convert vars to replacements --- kustomize/commands/commands.go | 2 +- kustomize/commands/edit/all.go | 7 +- kustomize/commands/edit/fix/convert.go | 345 ++++++ kustomize/commands/edit/fix/convert_test.go | 1079 +++++++++++++++++ kustomize/commands/edit/fix/fix.go | 69 +- kustomize/commands/edit/fix/fix_test.go | 11 +- .../internal/kustfile/kustomizationfile.go | 1 + .../kustfile/kustomizationfile_test.go | 1 + kustomize/go.mod | 1 + 9 files changed, 1504 insertions(+), 12 deletions(-) create mode 100644 kustomize/commands/edit/fix/convert.go create mode 100644 kustomize/commands/edit/fix/convert_test.go diff --git a/kustomize/commands/commands.go b/kustomize/commands/commands.go index ab91aa3558..85e117e391 100644 --- a/kustomize/commands/commands.go +++ b/kustomize/commands/commands.go @@ -49,7 +49,7 @@ See https://sigs.k8s.io/kustomize completion.NewCommand(), makeBuildCommand(fSys, stdOut), edit.NewCmdEdit( - fSys, pvd.GetFieldValidator(), pvd.GetResourceFactory()), + fSys, pvd.GetFieldValidator(), pvd.GetResourceFactory(), stdOut), create.NewCmdCreate(fSys, pvd.GetResourceFactory()), version.NewCmdVersion(stdOut), openapi.NewCmdOpenAPI(stdOut), diff --git a/kustomize/commands/edit/all.go b/kustomize/commands/edit/all.go index 8709a5775a..ed595efed1 100644 --- a/kustomize/commands/edit/all.go +++ b/kustomize/commands/edit/all.go @@ -4,6 +4,8 @@ package edit import ( + "io" + "github.com/spf13/cobra" "sigs.k8s.io/kustomize/api/filesys" "sigs.k8s.io/kustomize/api/ifc" @@ -19,7 +21,8 @@ import ( // NewCmdEdit returns an instance of 'edit' subcommand. func NewCmdEdit( - fSys filesys.FileSystem, v ifc.Validator, rf *resource.Factory) *cobra.Command { + fSys filesys.FileSystem, v ifc.Validator, rf *resource.Factory, + w io.Writer) *cobra.Command { c := &cobra.Command{ Use: "edit", Short: "Edits a kustomization file", @@ -46,7 +49,7 @@ func NewCmdEdit( fSys, kv.NewLoader(loader.NewFileLoaderAtCwd(fSys), v), v), - fix.NewCmdFix(fSys), + fix.NewCmdFix(fSys, w), remove.NewCmdRemove(fSys, v), listbuiltin.NewCmdListBuiltinPlugin(), ) diff --git a/kustomize/commands/edit/fix/convert.go b/kustomize/commands/edit/fix/convert.go new file mode 100644 index 0000000000..bb8a898e72 --- /dev/null +++ b/kustomize/commands/edit/fix/convert.go @@ -0,0 +1,345 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fix + +import ( + "bytes" + "fmt" + "path" + "strconv" + "strings" + + "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/resid" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/yaml" +) + +func ConvertVarsToReplacements(fSys filesys.FileSystem, k *types.Kustomization) error { + if k.Vars == nil { + return nil + } + + k.Resources = append(k.Resources, k.Bases...) + k.Replacements = []types.ReplacementField{} + + files, err := filesTouchedByKustomize(k, "", fSys) + if err != nil { + return err + } + + for _, v := range k.Vars { + repl := &types.Replacement{} + if err := addTargets(repl, v.Name, files, fSys); err != nil { + return err + } + copySourceFromVars(repl, v) + if err := setPlaceholderValue(v.Name, files, fSys); err != nil { + return err + } + k.Replacements = append(k.Replacements, types.ReplacementField{Replacement: *repl}) + } + k.Vars = nil + return nil +} + +var patchTarget = make(map[string]types.Patch) + +func filesTouchedByKustomize(k *types.Kustomization, filepath string, fSys filesys.FileSystem) ([]string, error) { + var result []string + for _, r := range k.Resources { + // first, try to read resource as a base/directory + files, err := fSys.ReadDir(r) + if err == nil && len(files) > 0 { + for _, file := range files { + if !stringInSlice(file, []string{ + "kustomization.yaml", + "kustomization.yml", + "Kustomization", + }) { + continue + } + + b, err := fSys.ReadFile(path.Join(r, file)) + if err != nil { + continue + } + + subKt := &types.Kustomization{} + if err := yaml.Unmarshal(b, subKt); err != nil { + return nil, err + } + paths, err := filesTouchedByKustomize(subKt, r, fSys) + if err != nil { + return nil, err + } + result = append(result, paths...) + } + + } + // read the resource as a file + result = append(result, path.Join(filepath, r)) + + } + + // aggregate all of the paths from the `patches` field + for _, p := range k.Patches { + if p.Path != "" { + patchPath := path.Join(filepath, p.Path) + result = append(result, patchPath) + patchTarget[patchPath] = p + } + } + return result, nil +} + +func copySourceFromVars(repl *types.Replacement, v types.Var) { + repl.Source = &types.SourceSelector{} + apiVersion := v.ObjRef.APIVersion + group, version := resid.ParseGroupVersion(apiVersion) + repl.Source.Gvk.Group = group + repl.Source.Gvk.Version = version + repl.Source.Gvk.Kind = v.ObjRef.Kind + repl.Source.Name = v.ObjRef.Name + repl.Source.Namespace = v.ObjRef.Namespace + repl.Source.FieldPath = v.FieldRef.FieldPath +} + +func addTargets(repl *types.Replacement, varName string, files []string, fSys filesys.FileSystem) error { + for _, file := range files { + nodes, err := getNodesFromFile(file, fSys) + if err != nil { + continue + } + for _, n := range nodes { + fieldPaths, options, err := findVarName(n, varName, []string{}) + if err != nil { + return fmt.Errorf("error with %s: %s", file, err.Error()) + } + targets, err := constructTargets(file, n, fieldPaths, options) + if err != nil { + return err + } + repl.Targets = append(repl.Targets, targets...) + } + } + return nil +} + +func getNodesFromFile(fileName string, fSys filesys.FileSystem) ([]*kyaml.RNode, error) { + b, err := fSys.ReadFile(fileName) + if err != nil { + return nil, err + } + out := &bytes.Buffer{} + r := kio.ByteReadWriter{ + Reader: bytes.NewBufferString(string(b)), + Writer: out, + KeepReaderAnnotations: true, + OmitReaderAnnotations: true, + } + return r.Read() +} + +func findVarName(node *kyaml.RNode, varName string, path []string) ([]string, []*types.FieldOptions, error) { + var fieldPaths []string + var options []*types.FieldOptions + + switch node.YNode().Kind { + + case kyaml.SequenceNode: + elements, err := node.Elements() + if err != nil { + return nil, nil, err + } + for i := range elements { + nextPathItem := strings.TrimSpace(strconv.Itoa(i)) + fieldPathsToAdd, optionsToAdd, err := findVarName(elements[i], + varName, append(path, nextPathItem)) + if err != nil { + return nil, nil, err + } + fieldPaths = append(fieldPaths, fieldPathsToAdd...) + options = append(options, optionsToAdd...) + } + + case kyaml.MappingNode: + err := node.VisitFields(func(n *kyaml.MapNode) error { + nextPathItem := strings.TrimSpace(n.Key.MustString()) + fieldPathsToAdd, optionsToAdd, err := findVarName(n.Value.Copy(), + varName, append(path, nextPathItem)) + if err != nil { + return err + } + fieldPaths = append(fieldPaths, fieldPathsToAdd...) + options = append(options, optionsToAdd...) + return nil + }) + if err != nil { + return nil, nil, err + } + + case kyaml.ScalarNode: + value := node.YNode().Value + varString := fmt.Sprintf("$(%s)", varName) + if strings.Contains(value, varString) { + fieldPaths = append(fieldPaths, strings.Join(path, ".")) + optionsToAdd, err := constructFieldOptions(value, varString) + if err != nil { + return nil, nil, err + } + options = append(options, optionsToAdd...) + } + } + + return fieldPaths, options, nil +} + +func constructFieldOptions(value string, varString string) ([]*types.FieldOptions, error) { + if value == varString { + return []*types.FieldOptions{{}}, nil + } + + var delimiter string + var index int + i := strings.Index(value, varString) + + // all array accesses here are safe because we know value != varString and + // that value contains varString, so len(value) > len(varString) + switch { + case i == 0: // prefix + delimiter = string(value[len(varString)]) + index = 0 + case (i + len(varString)) >= len(value): // suffix + delimiter = string(value[i-1]) + index = len(strings.Split(value, delimiter)) - 1 + default: // in the middle somewhere + pre := string(value[i-1]) + post := string(value[i+len(varString)]) + if pre != post { + return nil, fmt.Errorf("cannot convert all vars to replacements; %s is not delimited", varString) + } + delimiter = pre + index = indexOf(varString, strings.Split(value, delimiter)) + if index == -1 { + // this should never happen + return nil, fmt.Errorf("internal error: could not get index of var %s", varString) + } + } + return []*types.FieldOptions{{Delimiter: delimiter, Index: index}}, nil +} + +func constructTargets(file string, node *kyaml.RNode, fieldPaths []string, + options []*types.FieldOptions) ([]*types.TargetSelector, error) { + + if len(fieldPaths) != len(options) { + // this should never happen + return nil, fmt.Errorf("internal error: length of fieldPaths != length of fieldOptions") + } + + if patch, ok := patchTarget[file]; ok { + if !patch.Options["allowNameChange"] || !patch.Options["allowKindChange"] { + return writePatchTargets(patch, node, fieldPaths, options) + } + } + + var result []*types.TargetSelector + meta, metaErr := node.GetMeta() + + for i := range fieldPaths { + target := &types.TargetSelector{ + Select: &types.Selector{ + ResId: resid.ResId{ + Name: node.GetName(), + Namespace: node.GetNamespace(), + Gvk: resid.Gvk{ + Kind: node.GetKind(), + }, + }, + }, + FieldPaths: []string{fieldPaths[i]}, + } + if options[i].String() != "" { + target.Options = options[i] + } + if metaErr == nil { + if meta.TypeMeta.APIVersion != "" { + group, version := resid.ParseGroupVersion(meta.TypeMeta.APIVersion) + target.Select.ResId.Gvk.Group = group + target.Select.ResId.Gvk.Version = version + } + } + + result = append(result, target) + } + + return result, nil +} + +// if the var appears in a patch, this must be handled differently than a regular +// resource because a patch may be applied to multiple resources and the resulting +// resources may have different IDs than the patch +func writePatchTargets(patch types.Patch, node *kyaml.RNode, fieldPaths []string, + options []*types.FieldOptions) ([]*types.TargetSelector, error) { + + var result []*types.TargetSelector + selector := patch.Target.Copy() + + for i := range fieldPaths { + target := &types.TargetSelector{ + Select: &selector, + FieldPaths: []string{fieldPaths[i]}, + } + if options[i].String() != "" { + target.Options = options[i] + } + if patch.Options["allowNameChange"] { + target.Select.ResId.Name = node.GetName() + } + if patch.Options["allowKindChange"] { + target.Select.ResId.Kind = node.GetKind() + } + if node.GetNamespace() != "" { + target.Select.ResId.Namespace = node.GetNamespace() + } + result = append(result, target) + } + return result, nil +} + +func setPlaceholderValue(varName string, files []string, fSys filesys.FileSystem) error { + for _, filename := range files { + b, err := fSys.ReadFile(filename) + if err != nil { + continue + } + newFileContents := strings.ReplaceAll(string(b), fmt.Sprintf("$(%s)", varName), + fmt.Sprintf("%s_PLACEHOLDER", varName)) + err = fSys.WriteFile(filename, []byte(newFileContents)) + if err != nil { + return err + } + } + return nil +} + +func stringInSlice(elem string, slice []string) bool { + for i := range slice { + if slice[i] == elem { + return true + } + } + return false +} + +func indexOf(varName string, slice []string) int { + for i := range slice { + if slice[i] == varName { + return i + } + } + return -1 +} diff --git a/kustomize/commands/edit/fix/convert_test.go b/kustomize/commands/edit/fix/convert_test.go new file mode 100644 index 0000000000..c803a3c5f8 --- /dev/null +++ b/kustomize/commands/edit/fix/convert_test.go @@ -0,0 +1,1079 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fix + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/api/filesys" + testutils_test "sigs.k8s.io/kustomize/kustomize/v4/commands/internal/testutils" +) + +func TestFixVarsSimple(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Pod + name: my-pod + version: v1 +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsDelimiterPrefix(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME)/path +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + options: + delimiter: / + select: + kind: Pod + name: my-pod + version: v1 +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER/path +`, string(content)) +} + +func TestFixVarsDelimiterSuffix(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: path/$(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + options: + delimiter: / + index: 1 + select: + kind: Pod + name: my-pod + version: v1 +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: path/SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsDelimiter(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: my/path/$(SOME_SECRET_NAME)/secret/path +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + options: + delimiter: / + index: 2 + select: + kind: Pod + name: my-pod + version: v1 +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: my/path/SOME_SECRET_NAME_PLACEHOLDER/secret/path +`, string(content)) +} + +func TestFixVarsNotDelimited(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: var$(SOME_SECRET_NAME)/path +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + err := cmd.RunE(cmd, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "error with pod.yaml: cannot convert all vars to replacements; $(SOME_SECRET_NAME) is not delimited") +} + +func TestFixVarsWithPatchBasic(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + name: my-pod + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod + labels: + foo: $(SOME_SECRET_NAME) +spec: + containers: + - image: myimage + name: hello +`) + patch := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + fSys.WriteFile("patch.yaml", patch) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + name: my-pod + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - metadata.labels.foo + select: + kind: Pod + name: my-pod + version: v1 + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Pod + name: my-pod +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod + labels: + foo: SOME_SECRET_NAME_PLACEHOLDER +spec: + containers: + - image: myimage + name: hello +`, string(content)) + + content, err = fSys.ReadFile("patch.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsWithPatchDifferentName(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + name: my-pod + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod + labels: + foo: $(SOME_SECRET_NAME) +spec: + containers: + - image: myimage + name: hello +`) + patch := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + fSys.WriteFile("patch.yaml", patch) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + // the second replacement target is from the patch, and used the patch's target selector + // as the replacement's target selector. Note that the replacement target uses the pod + // name 'my-pod', and not the name provided in the patch. + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + name: my-pod + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - metadata.labels.foo + select: + kind: Pod + name: my-pod + version: v1 + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Pod + name: my-pod +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod + labels: + foo: SOME_SECRET_NAME_PLACEHOLDER +spec: + containers: + - image: myimage + name: hello +`, string(content)) + + content, err = fSys.ReadFile("patch.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsWithPatchNameChange(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + name: my-pod + options: + allowNameChange: true + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello +`) + patch := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + fSys.WriteFile("patch.yaml", patch) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + // The replacement target from the patch uses the patch's name because + // allowNameChange is set to true. + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- options: + allowNameChange: true + path: patch.yaml + target: + kind: Pod + name: my-pod + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Pod + name: doesnt-matter +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello +`, string(content)) + + content, err = fSys.ReadFile("patch.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsWithPatchMultipleResources(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-1 +spec: + containers: + - image: myimage + name: hello +--- +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-2 +spec: + containers: + - image: myimage + name: hello +`) + patch := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + fSys.WriteFile("patch.yaml", patch) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + // The replacement target targets all resources of type `Pod` because the + // patch targets all resources of type `Pod`. + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Pod +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-1 +spec: + containers: + - image: myimage + name: hello +--- +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-2 +spec: + containers: + - image: myimage + name: hello +`, string(content)) + + content, err = fSys.ReadFile("patch.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsWithPatchMultipleResourcesAndKindChange(t *testing.T) { + kustomization := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- path: patch.yaml + target: + kind: Pod + options: + allowKindChange: true + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-1 +spec: + containers: + - image: myimage + name: hello +--- +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-2 +spec: + containers: + - image: myimage + name: hello +`) + patch := []byte(` +apiVersion: v1 +kind: Custom +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomization) + fSys.WriteFile("pod.yaml", pod) + fSys.WriteFile("patch.yaml", patch) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + + // The replacement target from the patch uses the patch's Kind because + // allowKindChange is set to true. + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml + +patches: +- options: + allowKindChange: true + path: patch.yaml + target: + kind: Pod + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Custom +`, string(content)) + + content, err = fSys.ReadFile("pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-1 +spec: + containers: + - image: myimage + name: hello +--- +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-2 +spec: + containers: + - image: myimage + name: hello +`, string(content)) + + content, err = fSys.ReadFile("patch.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Custom +metadata: + name: doesnt-matter +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} + +func TestFixVarsWithOverlay(t *testing.T) { + kustomizationOverlay := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- base + +vars: +- name: SOME_SECRET_NAME + objref: + kind: Secret + name: my-secret + apiVersion: v1 +`) + kustomizationBase := []byte(` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml +`) + pod := []byte(` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: $(SOME_SECRET_NAME) +`) + + fSys := filesys.MakeFsInMemory() + testutils_test.WriteTestKustomizationWith(fSys, kustomizationOverlay) + fSys.WriteFile("base/pod.yaml", pod) + fSys.WriteFile("base/kustomization.yaml", kustomizationBase) + cmd := NewCmdFix(fSys, os.Stdout) + assert.NoError(t, cmd.Flags().Set("vars", "true")) + assert.NoError(t, cmd.RunE(cmd, nil)) + content, err := testutils_test.ReadTestKustomization(fSys) + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- base + +replacements: +- source: + kind: Secret + name: my-secret + version: v1 + targets: + - fieldPaths: + - spec.containers.0.env.0.value + select: + kind: Pod + name: my-pod + version: v1 +`, string(content)) + + content, err = fSys.ReadFile("base/kustomization.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- pod.yaml +`, string(content)) + + content, err = fSys.ReadFile("base/pod.yaml") + assert.NoError(t, err) + assert.Equal(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - image: myimage + name: hello + env: + - name: SECRET_TOKEN + value: SOME_SECRET_NAME_PLACEHOLDER +`, string(content)) +} diff --git a/kustomize/commands/edit/fix/fix.go b/kustomize/commands/edit/fix/fix.go index 1149e69088..0fa64b1a21 100644 --- a/kustomize/commands/edit/fix/fix.go +++ b/kustomize/commands/edit/fix/fix.go @@ -4,13 +4,24 @@ package fix import ( + "bytes" + "fmt" + "io" + "github.com/spf13/cobra" + "github.com/spf13/pflag" "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/kustomize/v4/commands/build" "sigs.k8s.io/kustomize/kustomize/v4/commands/internal/kustfile" ) +var flags struct { + vars bool +} + // NewCmdFix returns an instance of 'fix' subcommand. -func NewCmdFix(fSys filesys.FileSystem) *cobra.Command { +func NewCmdFix(fSys filesys.FileSystem, w io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "fix", Short: "Fix the missing fields in kustomization file", @@ -21,14 +32,19 @@ func NewCmdFix(fSys filesys.FileSystem) *cobra.Command { `, RunE: func(cmd *cobra.Command, args []string) error { - return RunFix(fSys) + return RunFix(fSys, w) }, } + AddFlagVars(cmd.Flags()) return cmd } // RunFix runs `fix` command -func RunFix(fSys filesys.FileSystem) error { +func RunFix(fSys filesys.FileSystem, w io.Writer) error { + var oldOutput bytes.Buffer + oldBuildCmd := build.NewCmdBuild(fSys, build.MakeHelp(konfig.ProgramName, "build"), &oldOutput) + oldBuildCmd.RunE(oldBuildCmd, nil) + mf, err := kustfile.NewKustomizationFile(fSys) if err != nil { return err @@ -43,5 +59,50 @@ func RunFix(fSys filesys.FileSystem) error { if err != nil { return err } - return mf.Write(m) + + if flags.vars { + err = ConvertVarsToReplacements(fSys, m) + if err != nil { + return err + } + fmt.Fprintln(w, ` +Fixed fields: + patchesJson6902 -> patches + commonLabels -> labels + vars -> replacements`) + + } else { + fmt.Fprintln(w, ` +Fixed fields: + patchesJson6902 -> patches + commonLabels -> labels + +To convert vars -> replacements, run the command `+"`kustomize edit fix --vars`"+` + +WARNING: Converting vars to replacements will potentially overwrite many resource files +and the resulting files may not produce the same output when `+"`kustomize build`"+` is run. +We recommend doing this in a clean git repository where the change is easy to undo.`) + } + + writeErr := mf.Write(m) + + var fixedOutput bytes.Buffer + fixedBuildCmd := build.NewCmdBuild(fSys, build.MakeHelp(konfig.ProgramName, "build"), &fixedOutput) + err = fixedBuildCmd.RunE(fixedBuildCmd, nil) + if err != nil { + fmt.Fprintf(w, "Warning: 'Fixed' kustomization now produces the error when running `kustomize build`: %s", err.Error()) + } else if fixedOutput.String() != oldOutput.String() { + fmt.Fprintf(w, "Warning: 'Fixed' kustomization now produces different output when running `kustomize build`:\n...%s...\n", fixedOutput.String()) + } + + return writeErr +} + +func AddFlagVars(set *pflag.FlagSet) { + set.BoolVar( + &flags.vars, + "vars", + false, // default + `If specified, kustomize will attempt to convert vars to replacements. +We recommend doing this in a clean git repository where the change is easy to undo.`) } diff --git a/kustomize/commands/edit/fix/fix_test.go b/kustomize/commands/edit/fix/fix_test.go index 68a0623de9..7b745ebf6b 100644 --- a/kustomize/commands/edit/fix/fix_test.go +++ b/kustomize/commands/edit/fix/fix_test.go @@ -4,6 +4,7 @@ package fix import ( + "os" "testing" "github.com/google/go-cmp/cmp" @@ -16,7 +17,7 @@ func TestFix(t *testing.T) { fSys := filesys.MakeFsInMemory() testutils_test.WriteTestKustomizationWith(fSys, []byte(`nameprefix: some-prefix-`)) - cmd := NewCmdFix(fSys) + cmd := NewCmdFix(fSys, os.Stdout) assert.NoError(t, cmd.RunE(cmd, nil)) content, err := testutils_test.ReadTestKustomization(fSys) @@ -54,7 +55,7 @@ patches: `) fSys := filesys.MakeFsInMemory() testutils_test.WriteTestKustomizationWith(fSys, kustomizationContentWithOutdatedPatchesFieldTitle) - cmd := NewCmdFix(fSys) + cmd := NewCmdFix(fSys, os.Stdout) assert.NoError(t, cmd.RunE(cmd, nil)) content, err := testutils_test.ReadTestKustomization(fSys) @@ -98,7 +99,7 @@ kind: Kustomization `) fSys := filesys.MakeFsInMemory() testutils_test.WriteTestKustomizationWith(fSys, kustomizationContentWithOutdatedPatchesFieldTitle) - cmd := NewCmdFix(fSys) + cmd := NewCmdFix(fSys, os.Stdout) assert.NoError(t, cmd.RunE(cmd, nil)) content, err := testutils_test.ReadTestKustomization(fSys) @@ -132,7 +133,7 @@ kind: Kustomization `) fSys := filesys.MakeFsInMemory() testutils_test.WriteTestKustomizationWith(fSys, kustomizationContentWithOutdatedCommonLabels) - cmd := NewCmdFix(fSys) + cmd := NewCmdFix(fSys, os.Stdout) assert.NoError(t, cmd.RunE(cmd, nil)) content, err := testutils_test.ReadTestKustomization(fSys) @@ -157,7 +158,7 @@ labels: fSys := filesys.MakeFsInMemory() testutils_test.WriteTestKustomizationWith(fSys, kustomizationContentWithOutdatedCommonLabels) - cmd := NewCmdFix(fSys) + cmd := NewCmdFix(fSys, os.Stdout) err := cmd.RunE(cmd, nil) assert.Error(t, err) assert.Equal(t, err.Error(), "label name 'foo' exists in both commonLabels and labels") diff --git a/kustomize/commands/internal/kustfile/kustomizationfile.go b/kustomize/commands/internal/kustfile/kustomizationfile.go index 6d16ef323a..3daedb42be 100644 --- a/kustomize/commands/internal/kustfile/kustomizationfile.go +++ b/kustomize/commands/internal/kustfile/kustomizationfile.go @@ -58,6 +58,7 @@ func determineFieldOrder() []string { "GeneratorOptions", "Vars", "Images", + "Replacements", "Replicas", "Configurations", "Generators", diff --git a/kustomize/commands/internal/kustfile/kustomizationfile_test.go b/kustomize/commands/internal/kustfile/kustomizationfile_test.go index 094ebf8a65..f2bca33c02 100644 --- a/kustomize/commands/internal/kustfile/kustomizationfile_test.go +++ b/kustomize/commands/internal/kustfile/kustomizationfile_test.go @@ -40,6 +40,7 @@ func TestFieldOrder(t *testing.T) { "GeneratorOptions", "Vars", "Images", + "Replacements", "Replicas", "Configurations", "Generators", diff --git a/kustomize/go.mod b/kustomize/go.mod index 630041c5fb..1db303232d 100644 --- a/kustomize/go.mod +++ b/kustomize/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/google/go-cmp v0.5.2 github.com/pkg/errors v0.9.1 + github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0