feat: allow specifying a remote config dependency from Google Cloud S…

…torage (#9057)

* feat: Introduce remote config dependency from Google Cloud Storage.

* feat: Adjust remote config dependency for Google Cloud Storage to be more inline with git.

Instead of copying the skaffold config from GCS the user specifies a source that signifies the bucket and directory structure to copy recursively. For example, specifying the source "gs://my-bucket/dir1/*" will copy all the files and subdirectories stored in "gs://my-bucket/dir1/". This requires the user to also specify the path to the skaffold config relative to the provided source. In the example mentioned, if the skaffold config was stored in "gs://my-bucket/dir1/configs/skaffold.yaml" then the path required would be "configs/skaffold.yaml".

* docs: Generate doc changes for GCS remote config dependency

* fix: unnecessary proto conversion fix

* fix: import order in couple files causing lint issues

* fix: only perform checks that disable gcs config syncs if the cache exists

* fix: lint issue related to generated proto

* undo generated proto changes since it broke all checks

* Remove redundant is remote dependency check

cmd/skaffold/app/cmd/flags.go

@@ -143,8 +143,8 @@ var flagRegistry = []Flag{
 	},
 	{
 		Name:          "remote-cache-dir",
-		Usage:         "Specify the location of the git repositories cache (default $HOME/.skaffold/repos)",
-		Value:         &opts.RepoCacheDir,
+		Usage:         "Specify the location of the remote cache (default $HOME/.skaffold/remote-cache)",
+		Value:         &opts.RemoteCacheDir,
 		DefValue:      "",
 		FlagAddMethod: "StringVar",
 		DefinedOn:     []string{"all"},

cmd/skaffold/app/cmd/inspect.go

@@ -25,7 +25,7 @@ var inspectFlags = struct {
 	filename          string
 	outFormat         string
 	modules           []string
-	repoCacheDir      string
+	remoteCacheDir    string
 	buildEnv          string
 	profiles          []string
 	profile           string
@@ -50,5 +50,5 @@ func NewCmdInspect() *cobra.Command {
 func cmdInspectFlags(f *pflag.FlagSet) {
 	f.StringVarP(&inspectFlags.filename, "filename", "f", "skaffold.yaml", "Path to the local Skaffold config file. Defaults to `skaffold.yaml`")
 	f.StringVarP(&inspectFlags.outFormat, "format", "o", "json", "Output format. One of: json(default)")
-	f.StringVar(&inspectFlags.repoCacheDir, "remote-cache-dir", "", "Specify the location of the remote git repositories cache (defaults to $HOME/.skaffold/repos)")
+	f.StringVar(&inspectFlags.remoteCacheDir, "remote-cache-dir", "", "Specify the location of the remote cache (defaults to $HOME/.skaffold/remote-cache)")
 }

cmd/skaffold/app/cmd/inspect_build_env.go

@@ -225,7 +225,7 @@ func cmdBuildEnvListFlags(f *pflag.FlagSet) {
 func printBuildEnvsListOptions() inspect.Options {
 	return inspect.Options{
 		Filename:          inspectFlags.filename,
-		RepoCacheDir:      inspectFlags.repoCacheDir,
+		RemoteCacheDir:    inspectFlags.remoteCacheDir,
 		OutFormat:         inspectFlags.outFormat,
 		Modules:           inspectFlags.modules,
 		Profiles:          inspectFlags.profiles,
@@ -235,11 +235,11 @@ func printBuildEnvsListOptions() inspect.Options {
 
 func localBuildEnvOptions() inspect.Options {
 	return inspect.Options{
-		Filename:     inspectFlags.filename,
-		RepoCacheDir: inspectFlags.repoCacheDir,
-		OutFormat:    inspectFlags.outFormat,
-		Modules:      inspectFlags.modules,
-		Profile:      inspectFlags.profile,
+		Filename:       inspectFlags.filename,
+		RemoteCacheDir: inspectFlags.remoteCacheDir,
+		OutFormat:      inspectFlags.outFormat,
+		Modules:        inspectFlags.modules,
+		Profile:        inspectFlags.profile,
 		BuildEnvOptions: inspect.BuildEnvOptions{
 			Push:             buildEnvFlags.push.Value(),
 			TryImportMissing: buildEnvFlags.tryImportMissing.Value(),
@@ -272,11 +272,11 @@ func gcbBuildEnvOptions() inspect.Options {
 
 func addClusterBuildEnvOptions() inspect.Options {
 	return inspect.Options{
-		Filename:     inspectFlags.filename,
-		RepoCacheDir: inspectFlags.repoCacheDir,
-		OutFormat:    inspectFlags.outFormat,
-		Modules:      inspectFlags.modules,
-		Profile:      inspectFlags.profile,
+		Filename:       inspectFlags.filename,
+		RemoteCacheDir: inspectFlags.remoteCacheDir,
+		OutFormat:      inspectFlags.outFormat,
+		Modules:        inspectFlags.modules,
+		Profile:        inspectFlags.profile,
 		BuildEnvOptions: inspect.BuildEnvOptions{
 			PullSecretPath:           buildEnvFlags.pullSecretPath,
 			PullSecretName:           buildEnvFlags.pullSecretName,

cmd/skaffold/app/cmd/inspect_execution_modes.go

@@ -43,7 +43,7 @@ func cmdExecutionModesList() *cobra.Command {
 func listExecutionModes(ctx context.Context, out io.Writer) error {
 	return executionModes.PrintExecutionModesList(ctx, out, inspect.Options{
 		Filename:          inspectFlags.filename,
-		RepoCacheDir:      inspectFlags.repoCacheDir,
+		RemoteCacheDir:    inspectFlags.remoteCacheDir,
 		OutFormat:         inspectFlags.outFormat,
 		Modules:           inspectFlags.modules,
 		Profiles:          inspectFlags.profiles,

cmd/skaffold/app/cmd/inspect_job_manifest_paths.go

@@ -66,7 +66,7 @@ func cmdJobManifestPathsModify() *cobra.Command {
 func listJobManifestPaths(ctx context.Context, out io.Writer) error {
 	return jobManifestPaths.PrintJobManifestPathsList(ctx, out, inspect.Options{
 		Filename:          inspectFlags.filename,
-		RepoCacheDir:      inspectFlags.repoCacheDir,
+		RemoteCacheDir:    inspectFlags.remoteCacheDir,
 		OutFormat:         inspectFlags.outFormat,
 		Modules:           inspectFlags.modules,
 		Profiles:          inspectFlags.profiles,

cmd/skaffold/app/cmd/inspect_modules.go

@@ -49,7 +49,7 @@ func cmdModulesList() *cobra.Command {
 func listModules(ctx context.Context, out io.Writer) error {
 	return modules.PrintModulesList(ctx, out, inspect.Options{
 		Filename:       inspectFlags.filename,
-		RepoCacheDir:   inspectFlags.repoCacheDir,
+		RemoteCacheDir: inspectFlags.remoteCacheDir,
 		OutFormat:      inspectFlags.outFormat,
 		ModulesOptions: inspect.ModulesOptions{IncludeAll: modulesFlags.includeAll},
 	})

cmd/skaffold/app/cmd/inspect_namespaces.go

@@ -58,7 +58,7 @@ func cmdNamespacesList() *cobra.Command {
 func listNamespaces(ctx context.Context, out io.Writer, args []string) error {
 	return namespaces.PrintNamespacesList(ctx, out, args[0], inspect.Options{
 		Filename:          inspectFlags.filename,
-		RepoCacheDir:      inspectFlags.repoCacheDir,
+		RemoteCacheDir:    inspectFlags.remoteCacheDir,
 		OutFormat:         inspectFlags.outFormat,
 		Modules:           inspectFlags.modules,
 		Profiles:          inspectFlags.profiles,

cmd/skaffold/app/cmd/inspect_profiles.go

@@ -46,7 +46,7 @@ func cmdProfilesList() *cobra.Command {
 func listProfiles(ctx context.Context, out io.Writer) error {
 	return profiles.PrintProfilesList(ctx, out, inspect.Options{
 		Filename:        inspectFlags.filename,
-		RepoCacheDir:    inspectFlags.repoCacheDir,
+		RemoteCacheDir:  inspectFlags.remoteCacheDir,
 		OutFormat:       inspectFlags.outFormat,
 		Modules:         inspectFlags.modules,
 		ProfilesOptions: inspect.ProfilesOptions{BuildEnv: inspect.BuildEnv(inspectFlags.buildEnv)},

cmd/skaffold/app/cmd/inspect_tests.go

@@ -45,7 +45,7 @@ func cmdTestsList() *cobra.Command {
 func listTests(ctx context.Context, out io.Writer) error {
 	return tests.PrintTestsList(ctx, out, inspect.Options{
 		Filename:          inspectFlags.filename,
-		RepoCacheDir:      inspectFlags.repoCacheDir,
+		RemoteCacheDir:    inspectFlags.remoteCacheDir,
 		OutFormat:         inspectFlags.outFormat,
 		Modules:           inspectFlags.modules,
 		Profiles:          inspectFlags.profiles,

cmd/skaffold/app/cmd/lint.go

@@ -47,10 +47,10 @@ func doLint(ctx context.Context, out io.Writer) error {
 		return err
 	}
 	return lint.Lint(ctx, out, lint.Options{
-		Filename:     opts.ConfigurationFile,
-		RepoCacheDir: opts.RepoCacheDir,
-		OutFormat:    outFormat,
-		Modules:      opts.ConfigurationFilter,
-		Profiles:     opts.Profiles,
+		Filename:       opts.ConfigurationFile,
+		RemoteCacheDir: opts.RemoteCacheDir,
+		OutFormat:      outFormat,
+		Modules:        opts.ConfigurationFilter,
+		Profiles:       opts.Profiles,
 	}, runCtx)
 }

docs-v2/content/en/docs/design/config.md

@@ -134,22 +134,25 @@ In imported configurations, files are resolved relative to the location of impor
 
 ### Remote config dependency
 
-The required skaffold config can live in a remote git repository:
+The required skaffold config can live in a remote git repository or in Google Cloud Storage:
 
 ```yaml
-apiVersion: skaffold/v2beta12
+apiVersion: skaffold/v4beta7
 kind: Config
 requires:
   - configs: ["cfg1", "cfg2"]
     git:
       repo: http://github.com/GoogleContainerTools/skaffold.git
       path: getting-started/skaffold.yaml
       ref: main
+  - configs: ["cfg3"]
+    googleCloudStorage:
+      source: gs://my-bucket/dir1/*
+      path: config/skaffold.yaml
 ```
 
-The environment variable `SKAFFOLD_REMOTE_CACHE_DIR` or flag `--remote-cache-dir` specifies the download location for all remote repos. If undefined then it defaults to `~/.skaffold/repos`. 
-The repo root directory name is a hash of the repo `uri` and the `branch/ref`.
-Every execution of a remote module resets the cached repo to the referenced ref. The default ref is `master`. If `master` is not defined then it defaults to `main`.
+The environment variable `SKAFFOLD_REMOTE_CACHE_DIR` or flag `--remote-cache-dir` specifies the download location for all remote dependency contents. If undefined then it defaults to `~/.skaffold/remote-cache`. The remote cache directory consists of subdirectories with the contents retrieved from the remote dependency. For git dependencies the subdirectory name is a hash of the repo `uri` and the `branch/ref`. For Google Cloud Storage dependencies the subdirectory name is a hash of the `source`.
+
 The remote config gets treated like a local config after substituting the path with the actual path in the cache directory.
 
 ### Profile Activation in required configs