From 218dcf3e2a3ba7421d31237408e10bf75dbc1038 Mon Sep 17 00:00:00 2001 From: Aaron Prindle Date: Thu, 23 Mar 2023 22:26:23 +0000 Subject: [PATCH] feat: add 'skaffold inspect jobManifestPath' and 'skaffold transform-schema jobManifestPath' commands --- cmd/skaffold/app/cmd/inspect.go | 2 +- .../app/cmd/inspect_job_manifest_paths.go | 85 ++++++++++ cmd/skaffold/app/cmd/inspect_namespaces.go | 2 + pkg/skaffold/inspect/jobManifestPaths/list.go | 59 +++++++ .../inspect/jobManifestPaths/list_test.go | 148 ++++++++++++++++++ .../inspect/jobManifestPaths/modify.go | 84 ++++++++++ .../inspect/jobManifestPaths/modify_test.go | 90 +++++++++++ pkg/skaffold/verify/docker/verify.go | 4 +- 8 files changed, 470 insertions(+), 4 deletions(-) create mode 100644 cmd/skaffold/app/cmd/inspect_job_manifest_paths.go create mode 100644 pkg/skaffold/inspect/jobManifestPaths/list.go create mode 100644 pkg/skaffold/inspect/jobManifestPaths/list_test.go create mode 100644 pkg/skaffold/inspect/jobManifestPaths/modify.go create mode 100644 pkg/skaffold/inspect/jobManifestPaths/modify_test.go diff --git a/cmd/skaffold/app/cmd/inspect.go b/cmd/skaffold/app/cmd/inspect.go index fa3767f8abd..8a6de4cf626 100644 --- a/cmd/skaffold/app/cmd/inspect.go +++ b/cmd/skaffold/app/cmd/inspect.go @@ -43,7 +43,7 @@ func NewCmdInspect() *cobra.Command { WithDescription("Helper commands for Cloud Code IDEs to interact with and modify skaffold configuration files."). WithPersistentFlagAdder(cmdInspectFlags). Hidden(). - WithCommands(cmdModules(), cmdProfiles(), cmdBuildEnv(), cmdTests(), cmdNamespaces()) + WithCommands(cmdModules(), cmdProfiles(), cmdBuildEnv(), cmdTests(), cmdNamespaces(), cmdJobManifestPaths()) } func cmdInspectFlags(f *pflag.FlagSet) { diff --git a/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go b/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go new file mode 100644 index 00000000000..296168d2918 --- /dev/null +++ b/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "context" + "errors" + "io" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/inspect" + jobManifestPaths "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/inspect/jobManifestPaths" + olog "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log" +) + +func cmdJobManifestPaths() *cobra.Command { + return NewCmd("jobManifestPaths"). + WithDescription("View skaffold jobManifestPath information defined in the specified skaffold configuration"). + WithCommands(cmdJobManifestPathsList(), cmdJobManifestPathsModify()) +} + +func cmdJobManifestPathsList() *cobra.Command { + return NewCmd("list"). + WithExample("Get list of jobManifestPaths", "inspect jobManifestPaths list --format json"). + WithExample("Get list of jobManifestPaths targeting a specific configuration", "inspect jobManifestPaths list --profile local --format json"). + WithDescription("Print the list of jobManifestPaths that would be run for a given configuration (default skaffold configuration, specific module, specific profile, etc)."). + WithFlagAdder(cmdJobManifestPathsListFlags). + NoArgs(listJobManifestPaths) +} + +func cmdJobManifestPathsModify() *cobra.Command { + return NewCmd("modify"). + WithExample("Modify the skaffold verify jobManifestPaths", "inspect jobManifestPaths list --format json"). + WithExample("Modify the jobManifestPaths targeting a specific configuration", "inspect jobManifestPaths modify --profile local --format json"). + WithDescription("Print the list of jobManifestPaths that would be run for a given configuration (default skaffold configuration, specific module, specific profile, etc)."). + WithCommonFlags(). + WithFlags([]*Flag{ + // TODO(aaron-prindle) vvv 2 commands use this, should add to common flags w/ those 2 commands added + {Value: &outputFile, Name: "output", DefValue: "", Usage: "File to write `inspect jobManifestPath modify` result"}, + }). + WithArgs(func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + olog.Entry(context.TODO()).Errorf("`jobManifestPaths modify` requires exactly one manifest file path argument") + return errors.New("`jobManifestPaths modify` requires exactly one manifest file path argument") + } + return nil + }, modifyJobManifestPaths) +} + +func listJobManifestPaths(ctx context.Context, out io.Writer) error { + return jobManifestPaths.PrintJobManifestPathsList(ctx, out, inspect.Options{ + Filename: inspectFlags.filename, + RepoCacheDir: inspectFlags.repoCacheDir, + OutFormat: inspectFlags.outFormat, + Modules: inspectFlags.modules, + Profiles: inspectFlags.profiles, + PropagateProfiles: inspectFlags.propagateProfiles, + }) +} + +func cmdJobManifestPathsListFlags(f *pflag.FlagSet) { + f.StringSliceVarP(&inspectFlags.profiles, "profile", "p", nil, `Profile names to activate`) + f.BoolVar(&inspectFlags.propagateProfiles, "propagate-profiles", true, `Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated.`) + f.StringSliceVarP(&inspectFlags.modules, "module", "m", nil, "Names of modules to filter target action by.") +} + +func modifyJobManifestPaths(ctx context.Context, out io.Writer, args []string) error { + return jobManifestPaths.Modify(ctx, out, opts, args[0], outputFile) +} diff --git a/cmd/skaffold/app/cmd/inspect_namespaces.go b/cmd/skaffold/app/cmd/inspect_namespaces.go index 822f3de7f80..1c0f0a3acf8 100644 --- a/cmd/skaffold/app/cmd/inspect_namespaces.go +++ b/cmd/skaffold/app/cmd/inspect_namespaces.go @@ -26,6 +26,7 @@ import ( "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/inspect" namespaces "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/inspect/namespaces" + olog "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log" ) func cmdNamespaces() *cobra.Command { @@ -42,6 +43,7 @@ func cmdNamespacesList() *cobra.Command { WithFlagAdder(cmdNamespacesListFlags). WithArgs(func(cmd *cobra.Command, args []string) error { if len(args) != 1 { + olog.Entry(context.TODO()).Errorf("`inspect namespaces list` requires exactly one manifest file path argument") return errors.New("`inspect namespaces list` requires exactly one manifest file path argument") } return nil diff --git a/pkg/skaffold/inspect/jobManifestPaths/list.go b/pkg/skaffold/inspect/jobManifestPaths/list.go new file mode 100644 index 00000000000..0b4bb0d6dc3 --- /dev/null +++ b/pkg/skaffold/inspect/jobManifestPaths/list.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inspect + +import ( + "context" + "io" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/inspect" +) + +type jobManifestPathList struct { + VerifyJobManifestPaths map[string]string `json:"verifyJobManifestPaths"` + CustomActionJobManifestPaths map[string]string `json:"customActionJobManifestPaths"` +} + +func PrintJobManifestPathsList(ctx context.Context, out io.Writer, opts inspect.Options) error { + formatter := inspect.OutputFormatter(out, opts.OutFormat) + cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ + ConfigurationFile: opts.Filename, + ConfigurationFilter: opts.Modules, + RepoCacheDir: opts.RepoCacheDir, + Profiles: opts.Profiles, + PropagateProfiles: opts.PropagateProfiles, + }) + if err != nil { + formatter.WriteErr(err) + return err + } + + l := &jobManifestPathList{ + VerifyJobManifestPaths: map[string]string{}, + CustomActionJobManifestPaths: map[string]string{}, + } + for _, c := range cfgs { + for _, tc := range c.Verify { + if tc.ExecutionMode.KubernetesClusterExecutionMode != nil && tc.ExecutionMode.KubernetesClusterExecutionMode.JobManifestPath != "" { + l.VerifyJobManifestPaths[tc.Name] = tc.ExecutionMode.KubernetesClusterExecutionMode.JobManifestPath + } + // TODO(#8572) add similar logic for customAction schema fields when they are complete + } + } + return formatter.Write(l) +} diff --git a/pkg/skaffold/inspect/jobManifestPaths/list_test.go b/pkg/skaffold/inspect/jobManifestPaths/list_test.go new file mode 100644 index 00000000000..4d8ec3e22e8 --- /dev/null +++ b/pkg/skaffold/inspect/jobManifestPaths/list_test.go @@ -0,0 +1,148 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inspect + +import ( + "bytes" + "context" + "errors" + "fmt" + "testing" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/inspect" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/parser" + sErrors "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/errors" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util/stringslice" + "github.com/GoogleContainerTools/skaffold/v2/testutil" +) + +func TestPrintJobManifestPathsList(t *testing.T) { + tests := []struct { + description string + profiles []string + module []string + err error + expected string + }{ + { + description: "print all jobManifestPaths where no jobManifestPath is set in the verify config", + expected: "{\"verifyJobManifestPaths\":{},\"customActionJobManifestPaths\":{}}" + "\n", + module: []string{"cfg-without-jobManifestPaths"}, + }, + { + description: "print all jobManifestPaths where one jobManifestPath is set in the verify config via a profile", + expected: "{\"verifyJobManifestPaths\":{\"foo\":\"foo.yaml\"},\"customActionJobManifestPaths\":{}}" + "\n", + profiles: []string{"has-jobManifestPath"}, + module: []string{"cfg-without-jobManifestPaths"}, + }, + { + description: "print all jobManifestPaths where one jobManifestPath is set in the verify config via a module", + expected: "{\"verifyJobManifestPaths\":{\"bar\":\"bar.yaml\"},\"customActionJobManifestPaths\":{}}" + "\n", + module: []string{"cfg-with-jobManifestPaths"}, + }, + { + description: "actionable error", + err: sErrors.MainConfigFileNotFoundErr("path/to/skaffold.yaml", fmt.Errorf("failed to read file : %q", "skaffold.yaml")), + expected: `{"errorCode":"CONFIG_FILE_NOT_FOUND_ERR","errorMessage":"unable to find configuration file \"path/to/skaffold.yaml\": failed to read file : \"skaffold.yaml\". Check that the specified configuration file exists at \"path/to/skaffold.yaml\"."}` + "\n", + }, + { + description: "generic error", + err: errors.New("some error occurred"), + expected: `{"errorCode":"INSPECT_UNKNOWN_ERR","errorMessage":"some error occurred"}` + "\n", + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + configSet := parser.SkaffoldConfigSet{ + &parser.SkaffoldConfigEntry{SkaffoldConfig: &latest.SkaffoldConfig{ + Metadata: latest.Metadata{Name: "cfg-without-jobManifestPaths"}, + Pipeline: latest.Pipeline{}, + Profiles: []latest.Profile{ + {Name: "has-jobManifestPath", + Pipeline: latest.Pipeline{ + Verify: []*latest.VerifyTestCase{ + { + Name: "foo", + Container: latest.VerifyContainer{ + Name: "foo", + Image: "foo", + }, + ExecutionMode: latest.VerifyExecutionModeConfig{ + VerifyExecutionModeType: latest.VerifyExecutionModeType{ + KubernetesClusterExecutionMode: &latest.KubernetesClusterVerifier{ + JobManifestPath: "foo.yaml", + }, + }, + }, + }, + }, + }, + }}, + }, SourceFile: "path/to/cfg-without-jobManifestPaths"}, + + &parser.SkaffoldConfigEntry{SkaffoldConfig: &latest.SkaffoldConfig{ + Metadata: latest.Metadata{Name: "cfg-with-jobManifestPaths"}, + Pipeline: latest.Pipeline{ + Verify: []*latest.VerifyTestCase{ + { + Name: "bar", + Container: latest.VerifyContainer{ + Name: "bar", + Image: "bar", + }, + ExecutionMode: latest.VerifyExecutionModeConfig{ + VerifyExecutionModeType: latest.VerifyExecutionModeType{ + KubernetesClusterExecutionMode: &latest.KubernetesClusterVerifier{ + JobManifestPath: "bar.yaml", + }, + }, + }, + }, + }, + }, + }, SourceFile: "path/to/cfg-with-default-namespace"}, + } + t.Override(&inspect.GetConfigSet, func(_ context.Context, opts config.SkaffoldOptions) (parser.SkaffoldConfigSet, error) { + // mock profile activation + var set parser.SkaffoldConfigSet + for _, c := range configSet { + if len(opts.ConfigurationFilter) > 0 && !stringslice.Contains(opts.ConfigurationFilter, c.Metadata.Name) { + continue + } + for _, pName := range opts.Profiles { + for _, profile := range c.Profiles { + if profile.Name != pName { + continue + } + c.Verify = profile.Verify + } + } + set = append(set, c) + } + return set, test.err + }) + var buf bytes.Buffer + err := PrintJobManifestPathsList(context.Background(), &buf, inspect.Options{ + OutFormat: "json", Modules: test.module, Profiles: test.profiles}) + t.CheckError(test.err != nil, err) + t.CheckDeepEqual(test.expected, buf.String()) + }) + } +} diff --git a/pkg/skaffold/inspect/jobManifestPaths/modify.go b/pkg/skaffold/inspect/jobManifestPaths/modify.go new file mode 100644 index 00000000000..e7628d1976b --- /dev/null +++ b/pkg/skaffold/inspect/jobManifestPaths/modify.go @@ -0,0 +1,84 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inspect + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/parser" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/yaml" +) + +var ( + getCfgs = parser.GetAllConfigs +) + +func Modify(ctx context.Context, out io.Writer, opts config.SkaffoldOptions, inputFile, outputFile string) error { + // Open our jsonFile + jsonFile, err := os.Open(inputFile) + if err != nil { + fmt.Println(err) + } + // defer the closing of our jsonFile so that we can parse it later on + defer jsonFile.Close() + + byteValue, _ := ioutil.ReadAll(jsonFile) + + var result jobManifestPathList + json.Unmarshal(byteValue, &result) + + // force absolute path resolution during modify + opts.MakePathsAbsolute = util.Ptr(true) + configs, err := getCfgs(ctx, opts) + if err != nil { + return err + } + if outputFile != "" { + f, err := os.Create(outputFile) + if err != nil { + return err + } + defer f.Close() + out = f + } + + // remove the dependency config references since they have already been imported and will be marshalled together. + for i := range configs { + configs[i].(*latest.SkaffoldConfig).Dependencies = nil + for j := range configs[i].(*latest.SkaffoldConfig).Verify { + if jobManifestPath, ok := result.VerifyJobManifestPaths[configs[i].(*latest.SkaffoldConfig).Verify[j].Name]; ok { + configs[i].(*latest.SkaffoldConfig).Verify[j].ExecutionMode.KubernetesClusterExecutionMode.JobManifestPath = jobManifestPath + } + } + } + + buf, err := yaml.MarshalWithSeparator(configs) + if err != nil { + return fmt.Errorf("marshalling configuration: %w", err) + } + out.Write(buf) + + return nil +} diff --git a/pkg/skaffold/inspect/jobManifestPaths/modify_test.go b/pkg/skaffold/inspect/jobManifestPaths/modify_test.go new file mode 100644 index 00000000000..4dc25400019 --- /dev/null +++ b/pkg/skaffold/inspect/jobManifestPaths/modify_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inspect + +import ( + "bytes" + "context" + "testing" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/util" + "github.com/GoogleContainerTools/skaffold/v2/testutil" +) + +func TestPrintJobManifestPathsModify(t *testing.T) { + tests := []struct { + description string + input string + shouldErr bool + expected string + }{ + { + description: "output", + input: "{\"verifyJobManifestPaths\":{\"foo\":\"modified-foo.yaml\"},\"customActionJobManifestPaths\":{}}", + expected: `apiVersion: skaffold/v4beta4 +kind: Config +verify: + - name: foo + container: + name: foo + image: foo + env: [] + executionMode: + kubernetesCluster: + jobManifestPath: modified-foo.yaml +`, + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + inputFile := t.TempFile("", []byte(test.input)) + + t.Override(&getCfgs, func(context.Context, config.SkaffoldOptions) ([]util.VersionedConfig, error) { + return []util.VersionedConfig{ + &latest.SkaffoldConfig{ + APIVersion: "skaffold/v4beta4", + Kind: "Config", + Pipeline: latest.Pipeline{ + Verify: []*latest.VerifyTestCase{ + { + Name: "foo", + Container: latest.VerifyContainer{ + Name: "foo", + Image: "foo", + }, + ExecutionMode: latest.VerifyExecutionModeConfig{ + VerifyExecutionModeType: latest.VerifyExecutionModeType{ + KubernetesClusterExecutionMode: &latest.KubernetesClusterVerifier{ + JobManifestPath: "foo.yaml", + }, + }, + }, + }, + }, + }, + }, + }, nil + }) + var b bytes.Buffer + err := Modify(context.Background(), &b, config.SkaffoldOptions{}, inputFile, "") + t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, b.String()) + }) + } +} diff --git a/pkg/skaffold/verify/docker/verify.go b/pkg/skaffold/verify/docker/verify.go index 0c893eabc61..0875f0e3f1a 100644 --- a/pkg/skaffold/verify/docker/verify.go +++ b/pkg/skaffold/verify/docker/verify.go @@ -218,10 +218,8 @@ func (v *Verifier) createAndRunContainer(ctx context.Context, out io.Writer, art opts.Bindings = bindings // verify waits for run to complete opts.Wait = true - // verify passes through os env to container env - - envVars := []string{} // adding in env vars from verify container schema field + envVars := []string{} for _, env := range tc.Container.Env { envVars = append(envVars, env.Name+"="+env.Value) }