diff --git a/api/internal/localizer/localizer.go b/api/internal/localizer/localizer.go index 150cefe2fe..181db13a20 100644 --- a/api/internal/localizer/localizer.go +++ b/api/internal/localizer/localizer.go @@ -118,12 +118,38 @@ func (lc *localizer) load() (*types.Kustomization, string, error) { // localizeNativeFields localizes paths on kustomize-native fields, like configMapGenerator, that kustomize has a // built-in understanding of. This excludes helm-related fields, such as `helmGlobals` and `helmCharts`. func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error { - for i, path := range kust.Components { - newPath, err := lc.localizeDir(path) + if path, exists := kust.OpenAPI["path"]; exists { + newPath, err := lc.localizeFile(path) if err != nil { - return errors.WrapPrefixf(err, "unable to localize components field") + return errors.WrapPrefixf(err, "unable to localize openapi path") + } + kust.OpenAPI["path"] = newPath + } + + for fieldName, field := range map[string]struct { + paths []string + locFn func(string) (string, error) + }{ + "components": { + kust.Components, + lc.localizeDir, + }, + "configurations": { + kust.Configurations, + lc.localizeFile, + }, + "crds": { + kust.Crds, + lc.localizeFile, + }, + } { + for i, path := range field.paths { + newPath, err := field.locFn(path) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize %s path", fieldName) + } + field.paths[i] = newPath } - kust.Components[i] = newPath } for i := range kust.ConfigMapGenerator { @@ -168,8 +194,7 @@ func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error { } } - // TODO(annasong): localize all other kustomization fields: resources, bases, crds, configurations, - // openapi, configMapGenerator.env, secretGenerator.env + // TODO(annasong): localize all other kustomization fields: resources, bases, configMapGenerator.env, secretGenerator.env return nil } diff --git a/api/internal/localizer/localizer_test.go b/api/internal/localizer/localizer_test.go index 61bc24885d..6817781879 100644 --- a/api/internal/localizer/localizer_test.go +++ b/api/internal/localizer/localizer_test.go @@ -103,6 +103,21 @@ func reportFSysDiff(t *testing.T, fSysExpected filesys.FileSystem, fSysActual fi require.NoError(t, err) } +func checkLocalizeInTargetSuccess(t *testing.T, files map[string]string) { + t.Helper() + + fSys := makeMemoryFs(t) + addFiles(t, fSys, "/a", files) + + err := Run("/a", "/", "dst", fSys) + require.NoError(t, err) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/a", files) + addFiles(t, fSysExpected, "/dst/a", files) + checkFSys(t, fSysExpected, fSys) +} + func TestTargetIsScope(t *testing.T) { kustomization := map[string]string{ "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 @@ -151,13 +166,7 @@ commonLabels: kind: Kustomization `, } - fSysExpected, fSysActual := makeFileSystems(t, "/a", kustomization) - - err := Run("/a", "/", "/dst", fSysActual) - require.NoError(t, err) - - addFiles(t, fSysExpected, "/dst/a", kustomization) - checkFSys(t, fSysExpected, fSysActual) + checkLocalizeInTargetSuccess(t, kustomization) } func TestLoadGVKNN(t *testing.T) { @@ -171,13 +180,7 @@ func TestLoadGVKNN(t *testing.T) { files := map[string]string{ "kustomization.yaml": kustomization, } - fSysExpected, fSysActual := makeFileSystems(t, "/a", files) - - err := Run("/a", "/a", "/dst", fSysActual) - require.NoError(t, err) - - addFiles(t, fSysExpected, "/dst", files) - checkFSys(t, fSysExpected, fSysActual) + checkLocalizeInTargetSuccess(t, files) }) } } @@ -197,25 +200,7 @@ imageTags: kind: Kustomization `, } - fSysExpected, fSysActual := makeFileSystems(t, "/alpha", kustomization) - - err := Run("/alpha", "/alpha", "/beta", fSysActual) - require.NoError(t, err) - - addFiles(t, fSysExpected, "/beta", map[string]string{ - "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 -bases: -- beta -configMapGenerator: -- env: env.properties -imageTags: -- name: postgres - newName: my-registry/my-postgres - newTag: v1 -kind: Kustomization -`, - }) - checkFSys(t, fSysExpected, fSysActual) + checkLocalizeInTargetSuccess(t, kustomization) } func TestLoadUnknownKustFields(t *testing.T) { @@ -247,13 +232,7 @@ patches: `, path), path: podConfiguration, } - expected, actual := makeFileSystems(t, "/a", kustAndPatch) - - err := Run("/a", "/", "/a/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/a/dst/a", kustAndPatch) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPatch) }) } } @@ -292,9 +271,9 @@ configMapGenerator: name: referenced-file kind: Kustomization `, - "env": "APPLE=orange", - "env.properties": "USERNAME=password", - "resource.yaml": podConfiguration, + "env": "APPLE=orange", + "env.properties": "USERNAME=password", + "dir/resource.yaml": podConfiguration, } expected, actual := makeFileSystems(t, "/alpha/beta", targetAndUnreferenced) @@ -326,25 +305,95 @@ patches: allowNameChange: true path: patch.yaml `, - "patch.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Deployment -metadata: - name: not-used -spec: - template: - spec: - containers: - - name: nginx - image: nginx:1.21.0 + "patch.yaml": podConfiguration, + } + checkLocalizeInTargetSuccess(t, kustAndPatch) +} + +func TestLocalizeOpenAPI(t *testing.T) { + type testCase struct { + name string + files map[string]string + } + for _, test := range []testCase{ + { + name: "no_path", + files: map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +openapi: + version: v1.20.4 `, + }, + }, + { + name: "path", + files: map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +openapi: + path: openapi.json +`, + "openapi.json": `{ + "definitions": { + "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": { + "properties": { + "name": { + "type": "string" + } + }, + "type": "object" + } + } +}`, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + checkLocalizeInTargetSuccess(t, test.files) + }) } - expected, actual := makeFileSystems(t, "/", kustAndPatch) +} - err := Run("/", "", "", actual) - require.NoError(t, err) +func TestLocalizeConfigurations(t *testing.T) { + kustAndConfigs := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +configurations: +- commonLabels.yaml +- namePrefix.yaml +kind: Kustomization +`, + "commonLabels.yaml": `commonLabels: +- path: new/path + create: true`, + "namePrefix.yaml": `namePrefix: +- version: v1 + path: metadata/name +- group: custom + path: metadata/name`, + } + checkLocalizeInTargetSuccess(t, kustAndConfigs) +} - addFiles(t, expected, "/localized", kustAndPatch) - checkFSys(t, expected, actual) +func TestLocalizeCrds(t *testing.T) { + kustAndCrds := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +crds: +- crd1.yaml +- crd2.yaml +kind: Kustomization +`, + "crd1.yaml": `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: controller.stable.example.com`, + "crd2.yaml": `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: crontabs.stable.example.com +scope: Cluster`, + } + checkLocalizeInTargetSuccess(t, kustAndCrds) } func TestLocalizePatchesJson(t *testing.T) { @@ -376,13 +425,7 @@ patchesJson6902: {"op": "remove", "path": "/some/existing/path"}, ]`, } - expected, actual := makeFileSystems(t, "/alpha/beta", kustAndPatches) - - err := Run("/alpha/beta", "/", "/beta", actual) - require.NoError(t, err) - - addFiles(t, expected, "/beta/alpha/beta", kustAndPatches) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPatches) } func TestLocalizePatchesSM(t *testing.T) { @@ -401,13 +444,7 @@ patchesStrategicMerge: `, "patch.yaml": podConfiguration, } - expected, actual := makeFileSystems(t, "/a", kustAndPatches) - - err := Run("/a", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPatches) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPatches) } func TestLocalizeReplacements(t *testing.T) { @@ -426,7 +463,7 @@ replacements: name: my-map `, "replacement.yaml": `source: - fieldPath: path.*.to.[some=field] + fieldPath: path.to.some.field kind: Pod options: delimiter: / @@ -434,19 +471,14 @@ targets: - fieldPaths: - config\.kubernetes\.io.annotations - second.path + - path.*.to.[some=field] reject: - group: apps version: v2 select: namespace: my`, } - expected, actual := makeFileSystems(t, "/a", kustAndReplacement) - - err := Run("/a", "/", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst/a", kustAndReplacement) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndReplacement) } func TestLocalizeConfigMapGenerator(t *testing.T) { @@ -471,13 +503,7 @@ metadata: IS_GLOBAL=true`, "key.properties": "value", } - expected, actual := makeFileSystems(t, "/a/b", kustAndData) - - err := Run("/a/b", "", "", actual) - require.NoError(t, err) - - addFiles(t, expected, "/localized-b", kustAndData) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndData) } func TestLocalizeSecretGenerator(t *testing.T) { @@ -504,13 +530,7 @@ secretGenerator: "b/value.properties": "dmFsdWU=", "b/value": "dmFsdWU=", } - expected, actual := makeFileSystems(t, "/a", kustAndData) - - err := Run("/a", "/", "/localized-a", actual) - require.NoError(t, err) - - addFiles(t, expected, "/localized-a/a", kustAndData) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndData) } func TestLocalizeFileNoFile(t *testing.T) { @@ -551,13 +571,7 @@ path: patchSM-two.yaml "patchSM-one.yaml": podConfiguration, "patchSM-two.yaml": podConfiguration, } - expected, actual := makeFileSystems(t, "/a", kustAndPlugins) - - err := Run("/a", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPlugins) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPlugins) } func TestLocalizeMultiplePluginsInEntry(t *testing.T) { @@ -581,13 +595,7 @@ transformers: "patchSM-one.yaml": podConfiguration, "patchSM-two.yaml": podConfiguration, } - expected, actual := makeFileSystems(t, "/a", kustAndPlugins) - - err := Run("/a", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPlugins) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPlugins) } func TestLocalizeCleanedPathInPath(t *testing.T) { @@ -649,13 +657,7 @@ metadata: name: map `, } - expected, actual := makeFileSystems(t, "/a", kustAndPlugins) - - err := Run("/a", "", "/alpha/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/alpha/dst", kustAndPlugins) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPlugins) } func TestLocalizeTransformersPatch(t *testing.T) { @@ -681,13 +683,7 @@ path: patchSM.yaml `, "patchSM.yaml": podConfiguration, } - expected, actual := makeFileSystems(t, "/a", kustAndPatches) - - err := Run("/a", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPatches) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPatches) } func TestLocalizeTransformersPatchJson(t *testing.T) { @@ -722,13 +718,7 @@ target: ] `, } - expected, actual := makeFileSystems(t, "/a", kustAndPatches) - - err := Run("/a", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPatches) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPatches) } func TestLocalizePluginsNoPaths(t *testing.T) { @@ -751,13 +741,7 @@ metadata: prefix: copy `, } - expected, actual := makeFileSystems(t, "/a", kustAndPlugins) - - err := Run("/a", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPlugins) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPlugins) } func TestLocalizeValidators(t *testing.T) { @@ -792,18 +776,12 @@ replacements: namespace: test targets: - fieldPaths: - - path + - path.*.to.[some=field] select: namespace: test `, } - expected, actual := makeFileSystems(t, "/", kustAndPlugin) - - err := Run("/", "", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst", kustAndPlugin) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndPlugin) } func TestLocalizeBuiltinPluginsNotResource(t *testing.T) { @@ -959,13 +937,7 @@ namespace: kustomize-namespace }, } { t.Run(tc.name, func(t *testing.T) { - expected, actual := makeFileSystems(t, "/alpha/beta/gamma", tc.files) - - err := Run("/alpha/beta/gamma", "/alpha/beta", "/dst", actual) - require.NoError(t, err) - - addFiles(t, expected, "/dst/gamma", tc.files) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, tc.files) }) } } @@ -1017,11 +989,5 @@ kind: Component nameSuffix: -test `, } - expected, actual := makeFileSystems(t, "/", kustAndComponents) - - err := Run("/", "", "", actual) - require.NoError(t, err) - - addFiles(t, expected, "/localized", kustAndComponents) - checkFSys(t, expected, actual) + checkLocalizeInTargetSuccess(t, kustAndComponents) } diff --git a/api/internal/localizer/locloader.go b/api/internal/localizer/locloader.go index 68ed745a9c..85f4b3b06b 100644 --- a/api/internal/localizer/locloader.go +++ b/api/internal/localizer/locloader.go @@ -92,7 +92,7 @@ func (ll *Loader) Load(path string) ([]byte, error) { if filepath.IsAbs(path) { return nil, errors.Errorf("absolute paths not yet supported in alpha: file path %q is absolute", path) } - if ll.local { + if !loader.IsRemoteFile(path) && ll.local { cleanPath := cleanFilePath(ll.fSys, filesys.ConfirmedDir(ll.Root()), path) cleanAbs := filepath.Join(ll.Root(), cleanPath) dir := filesys.ConfirmedDir(filepath.Dir(cleanAbs)) diff --git a/api/krusty/localizer/runner.go b/api/krusty/localizer/runner.go new file mode 100644 index 0000000000..a6d831d0e3 --- /dev/null +++ b/api/krusty/localizer/runner.go @@ -0,0 +1,15 @@ +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package localizer + +import ( + "sigs.k8s.io/kustomize/api/internal/localizer" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +// Run `kustomize localize`s files referenced by kustomization target in scope to destination newDir on fSys. +func Run(fSys filesys.FileSystem, target, scope, newDir string) error { + return errors.Wrap(localizer.Run(target, scope, newDir, fSys)) +} diff --git a/api/krusty/localizer/runner_test.go b/api/krusty/localizer/runner_test.go new file mode 100644 index 0000000000..bcd3759d55 --- /dev/null +++ b/api/krusty/localizer/runner_test.go @@ -0,0 +1,235 @@ +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package localizer_test + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + . "sigs.k8s.io/kustomize/api/internal/localizer" + "sigs.k8s.io/kustomize/api/krusty/localizer" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +const ( + customSchema = `{ + "definitions": { + "v1alpha1.MyCRD": { + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "template": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodTemplateSpec" + } + }, + "type": "object" + }, + "status": { + "properties": { + "success": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object", + "x-kubernetes-group-version-kind": [ + { + "group": "example.com", + "kind": "MyCRD", + "version": "v1alpha1" + }, + { + "group": "", + "kind": "MyCRD", + "version": "v1alpha1" + } + ] + }, + "io.k8s.api.core.v1.PodTemplateSpec": { + "properties": { + "metadata": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" + }, + "spec": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodSpec" + } + }, + "type": "object" + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": { + "properties": { + "name": { + "type": "string" + } + }, + "type": "object" + }, + "io.k8s.api.core.v1.PodSpec": { + "properties": { + "containers": { + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + }, + "type": "array", + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + } + }, + "type": "object" + }, + "io.k8s.api.core.v1.Container": { + "properties": { + "command": { + "items": { + "type": "string" + }, + "type": "array" + }, + "image": { + "type": "string" + }, + "name": { + "type": "string" + }, + "ports": { + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "containerPort", + "protocol" + ], + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "containerPort", + "x-kubernetes-patch-strategy": "merge" + } + }, + "type": "object" + }, + "io.k8s.api.core.v1.ContainerPort": { + "properties": { + "containerPort": { + "format": "int32", + "type": "integer" + }, + "name": { + "type": "string" + }, + "protocol": { + "type": "string" + } + }, + "type": "object" + } + } +} +` +) + +func prepareFs(t *testing.T, files map[string]string) (memoryFs filesys.FileSystem, actualFs filesys.FileSystem, testDir filesys.ConfirmedDir) { + t.Helper() + + memoryFs = filesys.MakeFsInMemory() + actualFs = filesys.MakeFsOnDisk() + + testDir, err := filesys.NewTmpConfirmedDir() + require.NoError(t, err) + + setupDir(t, memoryFs, testDir.String(), files) + setupDir(t, actualFs, testDir.String(), files) + + t.Cleanup(func() { + _ = actualFs.RemoveAll(testDir.String()) + }) + + return memoryFs, actualFs, testDir +} + +func setupDir(t *testing.T, targetFs filesys.FileSystem, parentDir string, files map[string]string) { + t.Helper() + + for file, content := range files { + require.NoError(t, targetFs.WriteFile(filepath.Join(parentDir, file), []byte(content))) + } +} + +func getLocFilePath(t *testing.T, pathFromTestdata []string) string { + t.Helper() + + localizedPathDirs := []string{LocalizeDir, "raw.githubusercontent.com", "kubernetes-sigs", "kustomize", + "kustomize", "v4.5.7", "api", "krusty", "testdata"} + return filepath.Join(append(localizedPathDirs, pathFromTestdata...)...) +} + +// checkFs checks fsActual, the real file system, against fsExpected, a file system in memory, for contents +// in directory walkDir. +func checkFs(t *testing.T, walkDir string, fsExpected filesys.FileSystem, fsActual filesys.FileSystem) { + t.Helper() + + err := fsExpected.Walk(walkDir, func(path string, info fs.FileInfo, err error) error { + require.NoError(t, err) + + if info.IsDir() { + require.DirExists(t, path) + } else { + require.FileExists(t, path) + + expectedContent, err := fsExpected.ReadFile(path) + require.NoError(t, err) + actualContent, err := fsActual.ReadFile(path) + require.NoError(t, err) + require.Equal(t, string(expectedContent), string(actualContent)) + } + return nil + }) + require.NoError(t, err) + + err = fsActual.Walk(walkDir, func(path string, info fs.FileInfo, err error) error { + require.NoError(t, err) + + // no symlinks yet + require.NotEqual(t, os.ModeSymlink, info.Mode()&os.ModeSymlink) + require.True(t, fsExpected.Exists(path)) + return nil + }) + require.NoError(t, err) +} + +func TestRemoteFile(t *testing.T) { + const kustf = `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +openapi: + path: %s +` + fsExpected, fsActual, testDir := prepareFs(t, map[string]string{ + "kustomization.yaml": fmt.Sprintf(kustf, `https://raw.githubusercontent.com/kubernetes-sigs/kustomize/kustomize/v4.5.7/api/krusty/testdata/customschema.json`), + }) + + dst := testDir.Join("dst") + err := localizer.Run(fsActual, testDir.String(), "", dst) + require.NoError(t, err) + + localizedPath := getLocFilePath(t, []string{"customschema.json"}) + setupDir(t, fsExpected, dst, map[string]string{ + "kustomization.yaml": fmt.Sprintf(kustf, localizedPath), + localizedPath: customSchema, + }) + checkFs(t, testDir.String(), fsExpected, fsActual) +}