diff --git a/cmd/skaffold/app/cmd/flags.go b/cmd/skaffold/app/cmd/flags.go index 0e82752e7b8..8975ea87011 100644 --- a/cmd/skaffold/app/cmd/flags.go +++ b/cmd/skaffold/app/cmd/flags.go @@ -143,8 +143,8 @@ var flagRegistry = []Flag{ }, { Name: "remote-cache-dir", - Usage: "Specify the location of the git repositories cache (default $HOME/.skaffold/repos)", - Value: &opts.RepoCacheDir, + Usage: "Specify the location of the remote cache (default $HOME/.skaffold/remote-cache)", + Value: &opts.RemoteCacheDir, DefValue: "", FlagAddMethod: "StringVar", DefinedOn: []string{"all"}, diff --git a/cmd/skaffold/app/cmd/inspect.go b/cmd/skaffold/app/cmd/inspect.go index edfb1a6b375..32bd485de63 100644 --- a/cmd/skaffold/app/cmd/inspect.go +++ b/cmd/skaffold/app/cmd/inspect.go @@ -25,7 +25,7 @@ var inspectFlags = struct { filename string outFormat string modules []string - repoCacheDir string + remoteCacheDir string buildEnv string profiles []string profile string @@ -50,5 +50,5 @@ func NewCmdInspect() *cobra.Command { func cmdInspectFlags(f *pflag.FlagSet) { f.StringVarP(&inspectFlags.filename, "filename", "f", "skaffold.yaml", "Path to the local Skaffold config file. Defaults to `skaffold.yaml`") f.StringVarP(&inspectFlags.outFormat, "format", "o", "json", "Output format. One of: json(default)") - f.StringVar(&inspectFlags.repoCacheDir, "remote-cache-dir", "", "Specify the location of the remote git repositories cache (defaults to $HOME/.skaffold/repos)") + f.StringVar(&inspectFlags.remoteCacheDir, "remote-cache-dir", "", "Specify the location of the remote cache (defaults to $HOME/.skaffold/remote-cache)") } diff --git a/cmd/skaffold/app/cmd/inspect_build_env.go b/cmd/skaffold/app/cmd/inspect_build_env.go index fc04b906e4c..ce47d45bc09 100644 --- a/cmd/skaffold/app/cmd/inspect_build_env.go +++ b/cmd/skaffold/app/cmd/inspect_build_env.go @@ -225,7 +225,7 @@ func cmdBuildEnvListFlags(f *pflag.FlagSet) { func printBuildEnvsListOptions() inspect.Options { return inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, Modules: inspectFlags.modules, Profiles: inspectFlags.profiles, @@ -235,11 +235,11 @@ func printBuildEnvsListOptions() inspect.Options { func localBuildEnvOptions() inspect.Options { return inspect.Options{ - Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, - OutFormat: inspectFlags.outFormat, - Modules: inspectFlags.modules, - Profile: inspectFlags.profile, + Filename: inspectFlags.filename, + RemoteCacheDir: inspectFlags.remoteCacheDir, + OutFormat: inspectFlags.outFormat, + Modules: inspectFlags.modules, + Profile: inspectFlags.profile, BuildEnvOptions: inspect.BuildEnvOptions{ Push: buildEnvFlags.push.Value(), TryImportMissing: buildEnvFlags.tryImportMissing.Value(), @@ -272,11 +272,11 @@ func gcbBuildEnvOptions() inspect.Options { func addClusterBuildEnvOptions() inspect.Options { return inspect.Options{ - Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, - OutFormat: inspectFlags.outFormat, - Modules: inspectFlags.modules, - Profile: inspectFlags.profile, + Filename: inspectFlags.filename, + RemoteCacheDir: inspectFlags.remoteCacheDir, + OutFormat: inspectFlags.outFormat, + Modules: inspectFlags.modules, + Profile: inspectFlags.profile, BuildEnvOptions: inspect.BuildEnvOptions{ PullSecretPath: buildEnvFlags.pullSecretPath, PullSecretName: buildEnvFlags.pullSecretName, diff --git a/cmd/skaffold/app/cmd/inspect_execution_modes.go b/cmd/skaffold/app/cmd/inspect_execution_modes.go index 3ba63eb7bb1..8d098b0a3c4 100644 --- a/cmd/skaffold/app/cmd/inspect_execution_modes.go +++ b/cmd/skaffold/app/cmd/inspect_execution_modes.go @@ -43,7 +43,7 @@ func cmdExecutionModesList() *cobra.Command { func listExecutionModes(ctx context.Context, out io.Writer) error { return executionModes.PrintExecutionModesList(ctx, out, inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, Modules: inspectFlags.modules, Profiles: inspectFlags.profiles, diff --git a/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go b/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go index 296168d2918..23ff8da4594 100644 --- a/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go +++ b/cmd/skaffold/app/cmd/inspect_job_manifest_paths.go @@ -66,7 +66,7 @@ func cmdJobManifestPathsModify() *cobra.Command { func listJobManifestPaths(ctx context.Context, out io.Writer) error { return jobManifestPaths.PrintJobManifestPathsList(ctx, out, inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, Modules: inspectFlags.modules, Profiles: inspectFlags.profiles, diff --git a/cmd/skaffold/app/cmd/inspect_modules.go b/cmd/skaffold/app/cmd/inspect_modules.go index d42d6ba6707..6a74aef647c 100644 --- a/cmd/skaffold/app/cmd/inspect_modules.go +++ b/cmd/skaffold/app/cmd/inspect_modules.go @@ -49,7 +49,7 @@ func cmdModulesList() *cobra.Command { func listModules(ctx context.Context, out io.Writer) error { return modules.PrintModulesList(ctx, out, inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, ModulesOptions: inspect.ModulesOptions{IncludeAll: modulesFlags.includeAll}, }) diff --git a/cmd/skaffold/app/cmd/inspect_namespaces.go b/cmd/skaffold/app/cmd/inspect_namespaces.go index 1c0f0a3acf8..acdad632795 100644 --- a/cmd/skaffold/app/cmd/inspect_namespaces.go +++ b/cmd/skaffold/app/cmd/inspect_namespaces.go @@ -58,7 +58,7 @@ func cmdNamespacesList() *cobra.Command { func listNamespaces(ctx context.Context, out io.Writer, args []string) error { return namespaces.PrintNamespacesList(ctx, out, args[0], inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, Modules: inspectFlags.modules, Profiles: inspectFlags.profiles, diff --git a/cmd/skaffold/app/cmd/inspect_profiles.go b/cmd/skaffold/app/cmd/inspect_profiles.go index d0781b1a72d..ff4b56142c5 100644 --- a/cmd/skaffold/app/cmd/inspect_profiles.go +++ b/cmd/skaffold/app/cmd/inspect_profiles.go @@ -46,7 +46,7 @@ func cmdProfilesList() *cobra.Command { func listProfiles(ctx context.Context, out io.Writer) error { return profiles.PrintProfilesList(ctx, out, inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, Modules: inspectFlags.modules, ProfilesOptions: inspect.ProfilesOptions{BuildEnv: inspect.BuildEnv(inspectFlags.buildEnv)}, diff --git a/cmd/skaffold/app/cmd/inspect_tests.go b/cmd/skaffold/app/cmd/inspect_tests.go index 44af006d487..2acebc1d695 100644 --- a/cmd/skaffold/app/cmd/inspect_tests.go +++ b/cmd/skaffold/app/cmd/inspect_tests.go @@ -45,7 +45,7 @@ func cmdTestsList() *cobra.Command { func listTests(ctx context.Context, out io.Writer) error { return tests.PrintTestsList(ctx, out, inspect.Options{ Filename: inspectFlags.filename, - RepoCacheDir: inspectFlags.repoCacheDir, + RemoteCacheDir: inspectFlags.remoteCacheDir, OutFormat: inspectFlags.outFormat, Modules: inspectFlags.modules, Profiles: inspectFlags.profiles, diff --git a/cmd/skaffold/app/cmd/lint.go b/cmd/skaffold/app/cmd/lint.go index bac209f423c..c62731dd152 100644 --- a/cmd/skaffold/app/cmd/lint.go +++ b/cmd/skaffold/app/cmd/lint.go @@ -47,10 +47,10 @@ func doLint(ctx context.Context, out io.Writer) error { return err } return lint.Lint(ctx, out, lint.Options{ - Filename: opts.ConfigurationFile, - RepoCacheDir: opts.RepoCacheDir, - OutFormat: outFormat, - Modules: opts.ConfigurationFilter, - Profiles: opts.Profiles, + Filename: opts.ConfigurationFile, + RemoteCacheDir: opts.RemoteCacheDir, + OutFormat: outFormat, + Modules: opts.ConfigurationFilter, + Profiles: opts.Profiles, }, runCtx) } diff --git a/docs-v2/content/en/docs/design/config.md b/docs-v2/content/en/docs/design/config.md index a81cdadf971..bf7e9013d2b 100644 --- a/docs-v2/content/en/docs/design/config.md +++ b/docs-v2/content/en/docs/design/config.md @@ -134,10 +134,10 @@ In imported configurations, files are resolved relative to the location of impor ### Remote config dependency -The required skaffold config can live in a remote git repository: +The required skaffold config can live in a remote git repository or in Google Cloud Storage: ```yaml -apiVersion: skaffold/v2beta12 +apiVersion: skaffold/v4beta7 kind: Config requires: - configs: ["cfg1", "cfg2"] @@ -145,11 +145,14 @@ requires: repo: http://github.com/GoogleContainerTools/skaffold.git path: getting-started/skaffold.yaml ref: main + - configs: ["cfg3"] + googleCloudStorage: + source: gs://my-bucket/dir1/* + path: config/skaffold.yaml ``` -The environment variable `SKAFFOLD_REMOTE_CACHE_DIR` or flag `--remote-cache-dir` specifies the download location for all remote repos. If undefined then it defaults to `~/.skaffold/repos`. -The repo root directory name is a hash of the repo `uri` and the `branch/ref`. -Every execution of a remote module resets the cached repo to the referenced ref. The default ref is `master`. If `master` is not defined then it defaults to `main`. +The environment variable `SKAFFOLD_REMOTE_CACHE_DIR` or flag `--remote-cache-dir` specifies the download location for all remote dependency contents. If undefined then it defaults to `~/.skaffold/remote-cache`. The remote cache directory consists of subdirectories with the contents retrieved from the remote dependency. For git dependencies the subdirectory name is a hash of the repo `uri` and the `branch/ref`. For Google Cloud Storage dependencies the subdirectory name is a hash of the `source`. + The remote config gets treated like a local config after substituting the path with the actual path in the cache directory. ### Profile Activation in required configs diff --git a/docs-v2/content/en/docs/references/cli/_index.md b/docs-v2/content/en/docs/references/cli/_index.md index 6563950db32..086ae96c49f 100644 --- a/docs-v2/content/en/docs/references/cli/_index.md +++ b/docs-v2/content/en/docs/references/cli/_index.md @@ -135,7 +135,7 @@ Options: -m, --module=[]: Filter Skaffold configs to only the provided named modules -n, --namespace='': Runs deployments in the specified namespace. When used with 'render' command, renders manifests contain the namespace -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC --status-check=: Wait for deployed resources to stabilize @@ -239,7 +239,7 @@ Options: --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. --push=: Push the built images to the specified image repository. -q, --quiet=false: Suppress the build output and print image built on success. See --output to format output. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC --skip-tests=false: Whether to skip the tests after building @@ -462,7 +462,7 @@ Options: --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. --protocols=[]: Priority sorted order of debugger protocols to support. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --resource-selector-rules-file='': Path to JSON file specifying the deny list of yaml objects for skaffold to NOT transform with 'image' and 'label' field replacements. NOTE: this list is additive to skaffold's default denylist and denylist has priority over allowlist --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC @@ -585,7 +585,7 @@ Options: -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories. Usage: @@ -669,7 +669,7 @@ The build result from a previous 'skaffold build --file-output' run can be used -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --resource-selector-rules-file='': Path to JSON file specifying the deny list of yaml objects for skaffold to NOT transform with 'image' and 'label' field replacements. NOTE: this list is additive to skaffold's default denylist and denylist has priority over allowlist --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC @@ -790,7 +790,7 @@ Options: -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --resource-selector-rules-file='': Path to JSON file specifying the deny list of yaml objects for skaffold to NOT transform with 'image' and 'label' field replacements. NOTE: this list is additive to skaffold's default denylist and denylist has priority over allowlist --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC @@ -909,7 +909,7 @@ Options: -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --sync-remote-cache='missing': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories. --yaml-only=false: Only prints the effective skaffold.yaml configuration @@ -974,7 +974,7 @@ The build result from a previous 'skaffold build --file-output' run can be used -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories. @@ -1030,7 +1030,7 @@ Options: -m, --module=[]: Filter Skaffold configs to only the provided named modules -o, --output='': File to write the changed config (instead of standard output) --overwrite=false: Overwrite original config with fixed config - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --sync-remote-cache='missing': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories. --version='skaffold/v4beta7': Target schema version to upgrade to @@ -1071,7 +1071,7 @@ Options: --generate-manifests=false: Allows skaffold to try and generate basic kubernetes resources to get your project started -k, --kubernetes-manifest=[]: A path or a glob pattern to kubernetes manifests (can be non-existent) to be added to the kubectl deployer (overrides detection of kubernetes manifests). Repeat the flag for multiple entries. E.g.: skaffold init -k pod.yaml -k k8s/*.yml -m, --module=[]: Filter Skaffold configs to only the provided named modules - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --skip-build=false: Skip generating build artifacts in Skaffold config --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories. @@ -1156,7 +1156,7 @@ The build result from a previous 'skaffold build --file-output' run can be used -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --resource-selector-rules-file='': Path to JSON file specifying the deny list of yaml objects for skaffold to NOT transform with 'image' and 'label' field replacements. NOTE: this list is additive to skaffold's default denylist and denylist has priority over allowlist --set=[]: overrides templated manifest fields by provided key-value pairs --set-value-file='': overrides templated manifest fields by a file containing key-value pairs in .env file format @@ -1250,7 +1250,7 @@ Options: -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --resource-selector-rules-file='': Path to JSON file specifying the deny list of yaml objects for skaffold to NOT transform with 'image' and 'label' field replacements. NOTE: this list is additive to skaffold's default denylist and denylist has priority over allowlist --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC @@ -1427,7 +1427,7 @@ The build result from a previous 'skaffold build --file-output' run can be used -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories. @@ -1491,7 +1491,7 @@ The build result from a previous 'skaffold build --file-output' run can be used -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation --propagate-profiles=true: Setting '--propagate-profiles=false' disables propagating profiles set by the '--profile' flag across config dependencies. This mean that only profiles defined directly in the target 'skaffold.yaml' file are activated. - --remote-cache-dir='': Specify the location of the git repositories cache (default $HOME/.skaffold/repos) + --remote-cache-dir='': Specify the location of the remote cache (default $HOME/.skaffold/remote-cache) --rpc-http-port=: tcp port to expose the Skaffold API over HTTP REST --rpc-port=: tcp port to expose the Skaffold API over gRPC --status-check=: Wait for deployed resources to stabilize diff --git a/docs-v2/content/en/schemas/v4beta7.json b/docs-v2/content/en/schemas/v4beta7.json index b26787073df..80b7b7c5ddf 100755 --- a/docs-v2/content/en/schemas/v4beta7.json +++ b/docs-v2/content/en/schemas/v4beta7.json @@ -1295,6 +1295,11 @@ "description": "describes a remote git repository containing the required configs.", "x-intellij-html-description": "describes a remote git repository containing the required configs." }, + "googleCloudStorage": { + "$ref": "#/definitions/GoogleCloudStorageInfo", + "description": "describes remote Google Cloud Storage objects containing the required configs.", + "x-intellij-html-description": "describes remote Google Cloud Storage objects containing the required configs." + }, "path": { "type": "string", "description": "describes the path to the file containing the required configs.", @@ -1305,6 +1310,7 @@ "configs", "path", "git", + "googleCloudStorage", "activeProfiles" ], "additionalProperties": false, @@ -2103,6 +2109,34 @@ "description": "*beta* describes how to do a remote build on [Google Cloud Build](https://cloud.google.com/cloud-build/docs/). Docker and Jib artifacts can be built on Cloud Build. The `projectId` needs to be provided and the currently logged in user should be given permissions to trigger new builds.", "x-intellij-html-description": "beta describes how to do a remote build on Google Cloud Build. Docker and Jib artifacts can be built on Cloud Build. The projectId needs to be provided and the currently logged in user should be given permissions to trigger new builds." }, + "GoogleCloudStorageInfo": { + "properties": { + "path": { + "type": "string", + "description": "relative path from the source to the skaffold configuration file. e.g. `configs/skaffold.yaml`.", + "x-intellij-html-description": "relative path from the source to the skaffold configuration file. e.g. configs/skaffold.yaml." + }, + "source": { + "type": "string", + "description": "Google Cloud Storage objects to copy. e.g. `gs://my-bucket/dir1/dir2/*`.", + "x-intellij-html-description": "Google Cloud Storage objects to copy. e.g. gs://my-bucket/dir1/dir2/*." + }, + "sync": { + "type": "boolean", + "description": "when set to `true` will reset the cached object to the latest remote version on every run.", + "x-intellij-html-description": "when set to true will reset the cached object to the latest remote version on every run." + } + }, + "preferredOrder": [ + "source", + "path", + "sync" + ], + "additionalProperties": false, + "type": "object", + "description": "contains information on the origin of skaffold configurations copied from Google Cloud Storage.", + "x-intellij-html-description": "contains information on the origin of skaffold configurations copied from Google Cloud Storage." + }, "Helm": { "required": [ "releases" diff --git a/examples/remote-multi-config-microservices/README.md b/examples/remote-multi-config-microservices/README.md index 304620f68ea..2d062fe9654 100644 --- a/examples/remote-multi-config-microservices/README.md +++ b/examples/remote-multi-config-microservices/README.md @@ -47,6 +47,6 @@ requires: ``` -There are two `git` dependencies from the same repository `GoogleContainerTools/skaffold`. You can add as many dependencies as you want across the same or different repositories; even between different branches of the same repository. Skaffold downloads each referenced repository (one copy per referenced branch) to its cache folder (`~/.skaffold/repos` by default). +There are two `git` dependencies from the same repository `GoogleContainerTools/skaffold`. You can add as many dependencies as you want across the same or different repositories; even between different branches of the same repository. Skaffold downloads each referenced repository (one copy per referenced branch) to its cache folder (`~/.skaffold/remote-cache` by default). The remote dependency caches should not be modified directly by the user. Skaffold will reset the cache to the latest from the remote on each run. diff --git a/integration/examples/remote-multi-config-microservices/README.md b/integration/examples/remote-multi-config-microservices/README.md index 304620f68ea..2d062fe9654 100644 --- a/integration/examples/remote-multi-config-microservices/README.md +++ b/integration/examples/remote-multi-config-microservices/README.md @@ -47,6 +47,6 @@ requires: ``` -There are two `git` dependencies from the same repository `GoogleContainerTools/skaffold`. You can add as many dependencies as you want across the same or different repositories; even between different branches of the same repository. Skaffold downloads each referenced repository (one copy per referenced branch) to its cache folder (`~/.skaffold/repos` by default). +There are two `git` dependencies from the same repository `GoogleContainerTools/skaffold`. You can add as many dependencies as you want across the same or different repositories; even between different branches of the same repository. Skaffold downloads each referenced repository (one copy per referenced branch) to its cache folder (`~/.skaffold/remote-cache` by default). The remote dependency caches should not be modified directly by the user. Skaffold will reset the cache to the latest from the remote on each run. diff --git a/pkg/skaffold/config/options.go b/pkg/skaffold/config/options.go index f3c2494638c..f135c9dcf1e 100644 --- a/pkg/skaffold/config/options.go +++ b/pkg/skaffold/config/options.go @@ -88,7 +88,7 @@ type SkaffoldOptions struct { DigestSource string Command string MinikubeProfile string - RepoCacheDir string + RemoteCacheDir string TransformRulesFile string VerifyDockerNetwork string VerifyEnvFile string diff --git a/pkg/skaffold/config/repo_cache.go b/pkg/skaffold/config/remote_cache.go similarity index 61% rename from pkg/skaffold/config/repo_cache.go rename to pkg/skaffold/config/remote_cache.go index 4de66c7bab3..ba6aebdab0a 100644 --- a/pkg/skaffold/config/repo_cache.go +++ b/pkg/skaffold/config/remote_cache.go @@ -16,15 +16,23 @@ limitations under the License. package config -import "errors" +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/mitchellh/go-homedir" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/constants" +) // These are the list of accepted values for flag `--sync-remote-cache`. const ( - // clone missing repositories and sync them on each skaffold run + // clone missing remote dependencies and sync them on each skaffold run always = "always" - // clone missing repositories but do not sync found repositories + // clone missing remote dependencies but do not sync found remote dependencies missing = "missing" - // do not clone missing repositories, and do not sync found repositories + // do not clone missing remote dependencies, and do not sync found remote dependencies never = "never" ) @@ -64,12 +72,26 @@ func (s *SyncRemoteCacheOption) String() string { return s.value } -// CloneDisabled specifies it cloning remote git repositories is disabled by flag value +// CloneDisabled specifies if cloning remote dependencies is disabled by flag value func (s *SyncRemoteCacheOption) CloneDisabled() bool { return s.value == never } -// FetchDisabled specifies it fetching remote git repositories is disabled by flag value +// FetchDisabled specifies if fetching remote dependencies is disabled by flag value func (s *SyncRemoteCacheOption) FetchDisabled() bool { return s.value == missing || s.value == never } + +// GetRemoteCacheDir returns the directory for the remote cache. +func GetRemoteCacheDir(opts SkaffoldOptions) (string, error) { + if opts.RemoteCacheDir != "" { + return opts.RemoteCacheDir, nil + } + + // cache location unspecified, use ~/.skaffold/remote-cache + home, err := homedir.Dir() + if err != nil { + return "", fmt.Errorf("retrieving home directory: %w", err) + } + return filepath.Join(home, constants.DefaultSkaffoldDir, "remote-cache"), nil +} diff --git a/pkg/skaffold/config/repo_cache_test.go b/pkg/skaffold/config/remote_cache_test.go similarity index 100% rename from pkg/skaffold/config/repo_cache_test.go rename to pkg/skaffold/config/remote_cache_test.go diff --git a/pkg/skaffold/gcs/gsutil.go b/pkg/skaffold/gcs/gsutil.go new file mode 100644 index 00000000000..722f2da0dfd --- /dev/null +++ b/pkg/skaffold/gcs/gsutil.go @@ -0,0 +1,125 @@ +/* +Copyright 2023 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcs + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + sErrors "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/errors" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util" + "github.com/GoogleContainerTools/skaffold/v2/proto/v1" +) + +const GsutilExec = "gsutil" + +type Gsutil struct{} + +// Copy calls `gsutil cp [-r] +func (g *Gsutil) Copy(ctx context.Context, src, dst string, recursive bool) error { + args := []string{"cp"} + if recursive { + args = append(args, "-r") + } + args = append(args, src, dst) + cmd := exec.CommandContext(ctx, GsutilExec, args...) + out, err := util.RunCmdOut(ctx, cmd) + if err != nil { + return fmt.Errorf("copy file(s) with %s failed: %w", GsutilExec, err) + } + log.Entry(ctx).Info(out) + return nil +} + +// SyncObjects syncs the target Google Cloud Storage objects with skaffold's local cache and returns the local path to the objects. +func SyncObjects(ctx context.Context, g latest.GoogleCloudStorageInfo, opts config.SkaffoldOptions) (string, error) { + remoteCacheDir, err := config.GetRemoteCacheDir(opts) + if err != nil { + return "", fmt.Errorf("failed determining remote cache directory: %w", err) + } + if err := os.MkdirAll(remoteCacheDir, 0700); err != nil { + return "", fmt.Errorf("failed creating remote cache directory: %w", err) + } + + sourceDir, err := getPerSourceDir(g) + if err != nil { + return "", fmt.Errorf("failed determining Google Cloud Storage remote cache directory for %q: %w", g.Source, err) + } + cacheDir := filepath.Join(remoteCacheDir, sourceDir) + if _, err := os.Stat(cacheDir); os.IsNotExist(err) { + // If cache doesn't exist and cloning is disabled then we can't move forward. + if opts.SyncRemoteCache.CloneDisabled() { + return "", syncDisabledErr(g, cacheDir) + } + // The subdirectory needs to exist to work with gsutil. + if err := os.MkdirAll(cacheDir, 0700); err != nil { + return "", fmt.Errorf("failed creating Google Cloud Storage cache directory for %q: %w", g.Source, err) + } + } else { + // If sync property is false then skip fetching latest object from remote storage. + if g.Sync != nil && !*g.Sync { + return cacheDir, nil + } + // If sync is turned off via flag `--sync-remote-cache` then skip fetching latest object from remote storage. + if opts.SyncRemoteCache.FetchDisabled() { + return cacheDir, nil + } + } + + gcs := Gsutil{} + if err := gcs.Copy(ctx, g.Source, cacheDir, true); err != nil { + return "", fmt.Errorf("failed to cache Google Cloud Storage objects from %q: %w", g.Source, err) + } + return cacheDir, nil +} + +// getPerSourceDir returns the directory used per Google Cloud Storage source. Directory is a hash of the source provided. +func getPerSourceDir(g latest.GoogleCloudStorageInfo) (string, error) { + inputs := []string{g.Source} + hasher := sha256.New() + enc := json.NewEncoder(hasher) + if err := enc.Encode(inputs); err != nil { + return "", err + } + + return base64.URLEncoding.EncodeToString(hasher.Sum(nil))[:32], nil +} + +// syncDisabledErr returns error to use when remote sync is turned off by the user and the Google Cloud Storage object doesn't exist inside the cache directory. +func syncDisabledErr(g latest.GoogleCloudStorageInfo, cacheDir string) error { + msg := fmt.Sprintf("cache directory %q for Google Cloud Storage source %q does not exist and remote cache sync is explicitly disabled via flag `--sync-remote-cache`", cacheDir, g.Source) + return sErrors.NewError(fmt.Errorf(msg), + &proto.ActionableErr{ + Message: msg, + ErrCode: proto.StatusCode_CONFIG_REMOTE_REPO_CACHE_NOT_FOUND_ERR, + Suggestions: []*proto.Suggestion{ + { + SuggestionCode: proto.SuggestionCode_CONFIG_ENABLE_REMOTE_REPO_SYNC, + Action: fmt.Sprintf("Either download the Google Cloud Storage objects manually to %q or set flag `--sync-remote-cache` to `always` or `missing`", cacheDir), + }, + }, + }) +} diff --git a/pkg/skaffold/gcs/gsutil_test.go b/pkg/skaffold/gcs/gsutil_test.go new file mode 100644 index 00000000000..87651bf4b9b --- /dev/null +++ b/pkg/skaffold/gcs/gsutil_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2023 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcs + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util" + "github.com/GoogleContainerTools/skaffold/v2/testutil" +) + +const ( + file = "source/file" + gcsFile = "gs://bucket/file" + folder = "source/" + gcsFolder = "gs://bucket/folder/" +) + +func TestCopy(t *testing.T) { + tests := []struct { + description string + src string + dst string + commands util.Command + recursive bool + shouldErr bool + }{ + { + description: "copy single file", + src: file, + dst: gcsFile, + commands: testutil.CmdRunOut(fmt.Sprintf("gsutil cp %s %s", file, gcsFile), "logs"), + }, + { + description: "copy recursively", + src: folder, + dst: gcsFolder, + commands: testutil.CmdRunOut(fmt.Sprintf("gsutil cp -r %s %s", folder, gcsFolder), "logs"), + recursive: true, + }, + { + description: "copy failed", + src: file, + dst: gcsFile, + commands: testutil.CmdRunOutErr(fmt.Sprintf("gsutil cp %s %s", file, gcsFile), "logs", fmt.Errorf("file not found")), + shouldErr: true, + }, + } + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + t.Override(&util.DefaultExecCommand, test.commands) + + gcs := Gsutil{} + err := gcs.Copy(context.Background(), test.src, test.dst, test.recursive) + + t.CheckError(test.shouldErr, err) + }) + } +} + +func TestSyncObject(t *testing.T) { + source := "gs://my-bucket/dir1/*" + path := "configs/skaffold.yaml" + sourceHash := "B7fSU6BUuHFJuLenty9ErOwyxPVqO5cB" + + tests := []struct { + description string + g latest.GoogleCloudStorageInfo + gsutilErr error + syncFlag string + existing bool + shouldErr bool + expected string + }{ + { + description: "first time copy succeeds", + g: latest.GoogleCloudStorageInfo{Source: source, Path: path}, + syncFlag: "always", + expected: sourceHash, + }, + { + description: "first time copy fails", + g: latest.GoogleCloudStorageInfo{Source: source, Path: path}, + gsutilErr: fmt.Errorf("not found"), + syncFlag: "always", + shouldErr: true, + }, + { + description: "first time copy with sync off via flag fails", + g: latest.GoogleCloudStorageInfo{Source: source, Path: path}, + syncFlag: "never", + shouldErr: true, + }, + { + description: "existing copy update succeeds", + g: latest.GoogleCloudStorageInfo{Source: source, Path: path}, + syncFlag: "always", + existing: true, + expected: sourceHash, + }, + { + description: "existing copy with sync off via flag succeeds", + g: latest.GoogleCloudStorageInfo{Source: source, Path: path}, + syncFlag: "never", + existing: true, + expected: sourceHash, + }, + { + description: "existing copy with sync off succeeds", + g: latest.GoogleCloudStorageInfo{Source: source, Path: path, Sync: util.Ptr(false)}, + syncFlag: "always", + existing: true, + expected: sourceHash, + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + td := t.NewTempDir() + if test.existing { + td.Touch(sourceHash) + } + + syncRemote := &config.SyncRemoteCacheOption{} + _ = syncRemote.Set(test.syncFlag) + opts := config.SkaffoldOptions{RemoteCacheDir: td.Root(), SyncRemoteCache: *syncRemote} + + var cmd *testutil.FakeCmd + if test.gsutilErr == nil { + cmd = testutil.CmdRunOut(fmt.Sprintf("gsutil cp -r %s %s", source, td.Path(sourceHash)), "logs") + } else { + cmd = testutil.CmdRunOutErr(fmt.Sprintf("gsutil cp -r %s %s", source, td.Path(sourceHash)), "logs", test.gsutilErr) + } + t.Override(&util.DefaultExecCommand, cmd) + + path, err := SyncObjects(context.Background(), test.g, opts) + var expected string + if !test.shouldErr { + expected = filepath.Join(td.Root(), test.expected) + } + t.CheckErrorAndDeepEqual(test.shouldErr, err, expected, path) + }) + } +} diff --git a/pkg/skaffold/git/gitutil.go b/pkg/skaffold/git/gitutil.go index 02c94531d8b..e7f3d21d24a 100644 --- a/pkg/skaffold/git/gitutil.go +++ b/pkg/skaffold/git/gitutil.go @@ -27,10 +27,7 @@ import ( "path/filepath" "strings" - "github.com/mitchellh/go-homedir" - "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" - "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/constants" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util" ) @@ -92,22 +89,8 @@ func getRepoDir(g latest.GitInfo) (string, error) { return base64.URLEncoding.EncodeToString(hasher.Sum(nil))[:32], nil } -// GetRepoCacheDir returns the directory for the remote git repo cache -func GetRepoCacheDir(opts config.SkaffoldOptions) (string, error) { - if opts.RepoCacheDir != "" { - return opts.RepoCacheDir, nil - } - - // cache location unspecified, use ~/.skaffold/repos - home, err := homedir.Dir() - if err != nil { - return "", fmt.Errorf("retrieving home directory: %w", err) - } - return filepath.Join(home, constants.DefaultSkaffoldDir, "repos"), nil -} - func syncRepo(ctx context.Context, g latest.GitInfo, opts config.SkaffoldOptions) (string, error) { - skaffoldCacheDir, err := GetRepoCacheDir(opts) + skaffoldCacheDir, err := config.GetRemoteCacheDir(opts) r := gitCmd{Dir: skaffoldCacheDir} if err != nil { return "", fmt.Errorf("failed to clone repo %s: %w", g.Repo, err) diff --git a/pkg/skaffold/git/gitutil_test.go b/pkg/skaffold/git/gitutil_test.go index de48fa9f377..ca74b7328d6 100644 --- a/pkg/skaffold/git/gitutil_test.go +++ b/pkg/skaffold/git/gitutil_test.go @@ -216,7 +216,7 @@ func TestSyncRepo(t *testing.T) { } syncRemote := &config.SyncRemoteCacheOption{} _ = syncRemote.Set(test.syncFlag) - opts := config.SkaffoldOptions{RepoCacheDir: td.Root(), SyncRemoteCache: *syncRemote} + opts := config.SkaffoldOptions{RemoteCacheDir: td.Root(), SyncRemoteCache: *syncRemote} var f *testutil.FakeCmd for _, v := range test.cmds { if f == nil { diff --git a/pkg/skaffold/inspect/buildEnv/add_cluster.go b/pkg/skaffold/inspect/buildEnv/add_cluster.go index 69a818e6978..9de628914ef 100644 --- a/pkg/skaffold/inspect/buildEnv/add_cluster.go +++ b/pkg/skaffold/inspect/buildEnv/add_cluster.go @@ -31,7 +31,7 @@ func AddClusterBuildEnv(ctx context.Context, out io.Writer, opts inspect.Options formatter := inspect.OutputFormatter(out, opts.OutFormat) cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, ConfigurationFilter: opts.Modules, SkipConfigDefaults: true, MakePathsAbsolute: util.Ptr(false), diff --git a/pkg/skaffold/inspect/buildEnv/add_gcb.go b/pkg/skaffold/inspect/buildEnv/add_gcb.go index 5e651650975..c6a12e371b9 100644 --- a/pkg/skaffold/inspect/buildEnv/add_gcb.go +++ b/pkg/skaffold/inspect/buildEnv/add_gcb.go @@ -32,7 +32,7 @@ func AddGcbBuildEnv(ctx context.Context, out io.Writer, opts inspect.Options) er formatter := inspect.OutputFormatter(out, opts.OutFormat) cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, ConfigurationFilter: opts.Modules, SkipConfigDefaults: true, MakePathsAbsolute: util.Ptr(false)}) diff --git a/pkg/skaffold/inspect/buildEnv/list.go b/pkg/skaffold/inspect/buildEnv/list.go index e2e02106520..5f3cf84d022 100644 --- a/pkg/skaffold/inspect/buildEnv/list.go +++ b/pkg/skaffold/inspect/buildEnv/list.go @@ -38,7 +38,7 @@ func PrintBuildEnvsList(ctx context.Context, out io.Writer, opts inspect.Options formatter := inspect.OutputFormatter(out, opts.OutFormat) cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, PropagateProfiles: opts.PropagateProfiles, ConfigurationFilter: opts.Modules, diff --git a/pkg/skaffold/inspect/executionModes/list.go b/pkg/skaffold/inspect/executionModes/list.go index 6058850b35e..63213819470 100644 --- a/pkg/skaffold/inspect/executionModes/list.go +++ b/pkg/skaffold/inspect/executionModes/list.go @@ -34,7 +34,7 @@ func PrintExecutionModesList(ctx context.Context, out io.Writer, opts inspect.Op cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, PropagateProfiles: opts.PropagateProfiles, }) diff --git a/pkg/skaffold/inspect/jobManifestPaths/list.go b/pkg/skaffold/inspect/jobManifestPaths/list.go index a358f9e2695..8590023e55c 100644 --- a/pkg/skaffold/inspect/jobManifestPaths/list.go +++ b/pkg/skaffold/inspect/jobManifestPaths/list.go @@ -34,7 +34,7 @@ func PrintJobManifestPathsList(ctx context.Context, out io.Writer, opts inspect. cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, PropagateProfiles: opts.PropagateProfiles, }) diff --git a/pkg/skaffold/inspect/modules/list.go b/pkg/skaffold/inspect/modules/list.go index 1b62733eaac..de4ce869caf 100644 --- a/pkg/skaffold/inspect/modules/list.go +++ b/pkg/skaffold/inspect/modules/list.go @@ -38,7 +38,7 @@ type moduleEntry struct { func PrintModulesList(ctx context.Context, out io.Writer, opts inspect.Options) error { formatter := inspect.OutputFormatter(out, opts.OutFormat) - cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ConfigurationFile: opts.Filename, RepoCacheDir: opts.RepoCacheDir}) + cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ConfigurationFile: opts.Filename, RemoteCacheDir: opts.RemoteCacheDir}) if err != nil { formatter.WriteErr(err) return err diff --git a/pkg/skaffold/inspect/namespaces/list.go b/pkg/skaffold/inspect/namespaces/list.go index a1aac459331..6a03a61b18f 100644 --- a/pkg/skaffold/inspect/namespaces/list.go +++ b/pkg/skaffold/inspect/namespaces/list.go @@ -88,7 +88,7 @@ func PrintNamespacesList(ctx context.Context, out io.Writer, manifestFile string cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, PropagateProfiles: opts.PropagateProfiles, }) diff --git a/pkg/skaffold/inspect/profiles/list.go b/pkg/skaffold/inspect/profiles/list.go index df7fff033c2..653bbd766d6 100644 --- a/pkg/skaffold/inspect/profiles/list.go +++ b/pkg/skaffold/inspect/profiles/list.go @@ -36,7 +36,7 @@ type profileEntry struct { func PrintProfilesList(ctx context.Context, out io.Writer, opts inspect.Options) error { formatter := inspect.OutputFormatter(out, opts.OutFormat) - cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, RepoCacheDir: opts.RepoCacheDir}) + cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, RemoteCacheDir: opts.RemoteCacheDir}) if err != nil { formatter.WriteErr(err) return err diff --git a/pkg/skaffold/inspect/tests/list.go b/pkg/skaffold/inspect/tests/list.go index 2be84695538..dfa6a752338 100644 --- a/pkg/skaffold/inspect/tests/list.go +++ b/pkg/skaffold/inspect/tests/list.go @@ -46,7 +46,7 @@ func PrintTestsList(ctx context.Context, out io.Writer, opts inspect.Options) er cfgs, err := inspect.GetConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, PropagateProfiles: opts.PropagateProfiles, }) diff --git a/pkg/skaffold/inspect/types.go b/pkg/skaffold/inspect/types.go index 529c9599401..288a023bb13 100644 --- a/pkg/skaffold/inspect/types.go +++ b/pkg/skaffold/inspect/types.go @@ -25,8 +25,8 @@ import ( type Options struct { // Filename is the `skaffold.yaml` file path Filename string - // RepoCacheDir is the directory for the remote git repository cache - RepoCacheDir string + // RemoteCacheDir is the directory for the remote cache + RemoteCacheDir string // OutFormat is the output format. One of: json OutFormat string // Modules is the module filter for specific commands diff --git a/pkg/skaffold/kubernetes/manifest/gcs.go b/pkg/skaffold/kubernetes/manifest/gcs.go index fbe81f2c817..565afa94481 100644 --- a/pkg/skaffold/kubernetes/manifest/gcs.go +++ b/pkg/skaffold/kubernetes/manifest/gcs.go @@ -23,7 +23,7 @@ import ( "path/filepath" "strings" - "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/gcs" ) var ManifestsFromGCS = "manifests_from_gcs" @@ -39,7 +39,7 @@ func DownloadFromGCS(manifests []string) (string, error) { if manifest == "" || !strings.HasPrefix(manifest, gcsPrefix) { return "", fmt.Errorf("%v is not a valid GCS path", manifest) } - gcs := util.Gsutil{} + gcs := gcs.Gsutil{} if err := gcs.Copy(context.Background(), manifest, dir, true); err != nil { return "", fmt.Errorf("failed to download manifests fom GCS: %w", err) } diff --git a/pkg/skaffold/kubernetes/manifest/util.go b/pkg/skaffold/kubernetes/manifest/util.go index a818cbd6035..77e4ec41d68 100644 --- a/pkg/skaffold/kubernetes/manifest/util.go +++ b/pkg/skaffold/kubernetes/manifest/util.go @@ -24,7 +24,7 @@ import ( "path/filepath" "strings" - "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/util" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/gcs" ) const ( @@ -51,7 +51,7 @@ func Write(manifests string, output string, manifestOut io.Writer) error { if err := dumpToFile(manifests, tempFile); err != nil { return err } - gcs := util.Gsutil{} + gcs := gcs.Gsutil{} if err := gcs.Copy(context.Background(), tempFile, output, false); err != nil { return writeErr(fmt.Errorf("failed to copy rendered manifests to GCS: %w", err)) } diff --git a/pkg/skaffold/lint/dockerfiles.go b/pkg/skaffold/lint/dockerfiles.go index ba5b32b4dfa..22956348d1a 100644 --- a/pkg/skaffold/lint/dockerfiles.go +++ b/pkg/skaffold/lint/dockerfiles.go @@ -111,7 +111,7 @@ func GetDockerfilesLintResults(ctx context.Context, opts Options, dockerCfg dock cfgs, err := getConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, }) if err != nil { diff --git a/pkg/skaffold/lint/k8smanifests.go b/pkg/skaffold/lint/k8smanifests.go index afe801c6977..ed025390774 100644 --- a/pkg/skaffold/lint/k8smanifests.go +++ b/pkg/skaffold/lint/k8smanifests.go @@ -53,7 +53,7 @@ func GetK8sManifestsLintResults(ctx context.Context, opts Options) (*[]Result, e cfgs, err := getConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, }) if err != nil { diff --git a/pkg/skaffold/lint/skaffoldyamls.go b/pkg/skaffold/lint/skaffoldyamls.go index 3fda31c02ae..e727a16142f 100644 --- a/pkg/skaffold/lint/skaffoldyamls.go +++ b/pkg/skaffold/lint/skaffoldyamls.go @@ -152,7 +152,7 @@ func GetSkaffoldYamlsLintResults(ctx context.Context, opts Options) (*[]Result, cfgs, err := getConfigSet(ctx, config.SkaffoldOptions{ ConfigurationFile: opts.Filename, ConfigurationFilter: opts.Modules, - RepoCacheDir: opts.RepoCacheDir, + RemoteCacheDir: opts.RemoteCacheDir, Profiles: opts.Profiles, }) if err != nil { diff --git a/pkg/skaffold/lint/types.go b/pkg/skaffold/lint/types.go index bcb525fc14f..7cfe745c91a 100644 --- a/pkg/skaffold/lint/types.go +++ b/pkg/skaffold/lint/types.go @@ -30,8 +30,8 @@ import ( type Options struct { // Filename is the `skaffold.yaml` file path Filename string - // RepoCacheDir is the directory for the remote git repository cache - RepoCacheDir string + // RemoteCacheDir is the directory for the remote cache + RemoteCacheDir string // OutFormat is the output format. One of: json OutFormat string // Modules is the module filter for specific commands diff --git a/pkg/skaffold/lsp/handler.go b/pkg/skaffold/lsp/handler.go index 6766ed36512..2577cc5a3e2 100644 --- a/pkg/skaffold/lsp/handler.go +++ b/pkg/skaffold/lsp/handler.go @@ -289,11 +289,11 @@ func lintFiles(ctx context.Context, runCtx docker.Config, // TODO(aaron-prindle) currently lint checks only filesystem, instead need to check VFS w/ documentManager info // need to make sure something like k8a-manifest.yaml comes from afero VFS and not os FS always results, err := lint.GetAllLintResults(ctx, lint.Options{ - Filename: opts.ConfigurationFile, - RepoCacheDir: opts.RepoCacheDir, - OutFormat: lint.PlainTextOutput, - Modules: opts.ConfigurationFilter, - Profiles: opts.Profiles, + Filename: opts.ConfigurationFile, + RemoteCacheDir: opts.RemoteCacheDir, + OutFormat: lint.PlainTextOutput, + Modules: opts.ConfigurationFilter, + Profiles: opts.Profiles, }, runCtx) if err != nil { diff --git a/pkg/skaffold/parser/config.go b/pkg/skaffold/parser/config.go index 7a9354ff855..aa715344b87 100644 --- a/pkg/skaffold/parser/config.go +++ b/pkg/skaffold/parser/config.go @@ -26,6 +26,7 @@ import ( "strings" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/config" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/gcs" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/git" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/parser/configlocations" @@ -50,6 +51,8 @@ type configOpts struct { isRequired bool // is this config resolved as a dependency as opposed to being set explicitly (via the `-f` flag) isDependency bool + // is this a remote config. + isRemote bool } // record captures the state of referenced configs. @@ -57,11 +60,12 @@ type record struct { appliedProfiles map[string]string // config -> list of applied profiles configNameToFile map[string]string // configName -> file path cachedRepos map[string]interface{} // git repo -> cache path or error + cachedObjects map[string]interface{} // google cloud storage object -> cache path or error allProfiles []string // list of all profiles, from all configuration files } func newRecord() *record { - return &record{appliedProfiles: make(map[string]string), configNameToFile: make(map[string]string), cachedRepos: make(map[string]interface{})} + return &record{appliedProfiles: make(map[string]string), configNameToFile: make(map[string]string), cachedRepos: make(map[string]interface{}), cachedObjects: make(map[string]interface{})} } // GetAllConfigs returns the list of all skaffold configurations parsed from the target config file in addition to all resolved dependency configs. @@ -80,7 +84,7 @@ func GetAllConfigs(ctx context.Context, opts config.SkaffoldOptions) ([]schemaUt // GetConfigSet returns the list of all skaffold configurations parsed from the target config file in addition to all resolved dependency configs as a `SkaffoldConfigSet`. // This struct additionally contains the file location that each skaffold configuration is parsed from. func GetConfigSet(ctx context.Context, opts config.SkaffoldOptions) (SkaffoldConfigSet, error) { - cOpts := configOpts{file: opts.ConfigurationFile, selection: nil, profiles: opts.Profiles, isRequired: false, isDependency: false} + cOpts := configOpts{file: opts.ConfigurationFile, selection: nil, profiles: opts.Profiles, isRequired: false, isDependency: false, isRemote: false} r := newRecord() cfgs, fieldsOverrodeByProfile, err := getConfigs(ctx, cOpts, opts, r) if err != nil { @@ -231,6 +235,7 @@ func processEachConfig(ctx context.Context, config *latest.SkaffoldConfig, cfgOp } } } + // These configOpts are overwritten by the processEachDependency function. newOpts := configOpts{file: cfgOpts.file, profiles: depProfiles, isRequired: required, isDependency: cfgOpts.isDependency} depConfigs, err := processEachDependency(ctx, d, newOpts, opts, r) if err != nil { @@ -240,16 +245,12 @@ func processEachConfig(ctx context.Context, config *latest.SkaffoldConfig, cfgOp } if required { - isRemote, err := isRemoteConfig(cfgOpts.file, opts) - if err != nil { - return nil, err - } configs = append(configs, &SkaffoldConfigEntry{ SkaffoldConfig: config, SourceFile: cfgOpts.file, SourceIndex: index, IsRootConfig: !cfgOpts.isDependency, - IsRemote: isRemote, + IsRemote: cfgOpts.isRemote, }) } return configs, nil @@ -277,12 +278,22 @@ func filterActiveProfiles(d latest.ConfigDependency, profiles []string) []string func processEachDependency(ctx context.Context, d latest.ConfigDependency, cfgOpts configOpts, opts config.SkaffoldOptions, r *record) (SkaffoldConfigSet, error) { path := makeConfigPathAbsolute(d.Path, cfgOpts.file) + isRemoteCfg := false if d.GitRepo != nil { cachePath, err := cacheRepo(ctx, *d.GitRepo, opts, r) if err != nil { return nil, sErrors.ConfigParsingError(fmt.Errorf("caching remote dependency %s: %w", d.GitRepo.Repo, err)) } path = cachePath + isRemoteCfg = true + } + if d.GoogleCloudStorage != nil { + cachePath, err := cacheGCSObject(ctx, *d.GoogleCloudStorage, opts, r) + if err != nil { + return nil, sErrors.ConfigParsingError(fmt.Errorf("caching remote dependency %s: %w", d.GoogleCloudStorage.Path, err)) + } + path = cachePath + isRemoteCfg = true } if path == "" { @@ -307,6 +318,7 @@ func processEachDependency(ctx context.Context, d latest.ConfigDependency, cfgOp cfgOpts.isDependency = cfgOpts.isDependency || path != cfgOpts.file cfgOpts.file = path cfgOpts.selection = d.Names + cfgOpts.isRemote = isRemoteCfg depConfigs, _, err := getConfigs(ctx, cfgOpts, opts, r) if err != nil { return nil, err @@ -338,6 +350,29 @@ func cacheRepo(ctx context.Context, g latest.GitInfo, opts config.SkaffoldOption } } +// cacheGCSObject downloads the referenced Google Cloud Storage object to skaffold's cache if required and returns the path to the target configuration file. +func cacheGCSObject(ctx context.Context, g latest.GoogleCloudStorageInfo, opts config.SkaffoldOptions, r *record) (string, error) { + key := g.Source + if p, found := r.cachedObjects[key]; found { + switch v := p.(type) { + case string: + return filepath.Join(v, g.Path), nil + case error: + return "", v + default: + log.Entry(context.TODO()).Fatalf("unable to check download status of Google Cloud Storage objects at %s", g.Source) + return "", nil + } + } + p, err := gcs.SyncObjects(ctx, g, opts) + if err != nil { + r.cachedObjects[key] = err + return "", err + } + r.cachedObjects[key] = p + return filepath.Join(p, g.Path), nil +} + // checkRevisit ensures that each config is activated with the same set of active profiles // It returns true if this config was visited once before. It additionally returns an error if the previous visit was with a different set of active profiles. func checkRevisit(config *latest.SkaffoldConfig, profiles []string, appliedProfiles map[string]string, file string, required bool, index int) (bool, error) { @@ -393,12 +428,3 @@ func getBase(cfgOpts configOpts) (string, error) { log.Entry(context.TODO()).Tracef("found cwd as base for absolute path substitution within skaffold config %s", cfgOpts.file) return util.RealWorkDir() } - -func isRemoteConfig(file string, opts config.SkaffoldOptions) (bool, error) { - dir, err := git.GetRepoCacheDir(opts) - if err != nil { - // ignore - return false, err - } - return strings.HasPrefix(file, dir), nil -} diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go index 163da9c67b4..c7347fb98bf 100644 --- a/pkg/skaffold/schema/latest/config.go +++ b/pkg/skaffold/schema/latest/config.go @@ -109,6 +109,18 @@ type GitInfo struct { Sync *bool `yaml:"sync,omitempty"` } +// GoogleCloudStorageInfo contains information on the origin of skaffold configurations copied from Google Cloud Storage. +type GoogleCloudStorageInfo struct { + // Source is the Google Cloud Storage objects to copy. e.g. `gs://my-bucket/dir1/dir2/*`. + Source string `yaml:"source,omitempty"` + + // Path is the relative path from the source to the skaffold configuration file. e.g. `configs/skaffold.yaml`. + Path string `yaml:"path,omitempty"` + + // Sync when set to `true` will reset the cached object to the latest remote version on every run. + Sync *bool `yaml:"sync,omitempty"` +} + // ConfigDependency describes a dependency on another skaffold configuration. type ConfigDependency struct { // Names includes specific named configs within the file path. If empty, then all configs in the file are included. @@ -120,6 +132,9 @@ type ConfigDependency struct { // GitRepo describes a remote git repository containing the required configs. GitRepo *GitInfo `yaml:"git,omitempty" yamltags:"oneOf=paths"` + // GoogleCloudStorage describes remote Google Cloud Storage objects containing the required configs. + GoogleCloudStorage *GoogleCloudStorageInfo `yaml:"googleCloudStorage,omitempty" yamltags:"oneOf=paths"` + // ActiveProfiles describes the list of profiles to activate when resolving the required configs. These profiles must exist in the imported config. ActiveProfiles []ProfileDependency `yaml:"activeProfiles,omitempty"` } diff --git a/pkg/skaffold/util/gsutil.go b/pkg/skaffold/util/gsutil.go deleted file mode 100644 index 5a47b710379..00000000000 --- a/pkg/skaffold/util/gsutil.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2020 The Skaffold Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "fmt" - "os/exec" - - "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log" -) - -const GsutilExec = "gsutil" - -type Gsutil struct{} - -// Copy calls `gsutil cp [-r] -func (g *Gsutil) Copy(ctx context.Context, src, dst string, recursive bool) error { - args := []string{"cp"} - if recursive { - args = append(args, "-r") - } - args = append(args, src, dst) - cmd := exec.CommandContext(ctx, GsutilExec, args...) - out, err := RunCmdOut(ctx, cmd) - if err != nil { - return fmt.Errorf("copy file(s) with %s failed: %w", GsutilExec, err) - } - log.Entry(ctx).Info(out) - return nil -} diff --git a/pkg/skaffold/util/gsutil_test.go b/pkg/skaffold/util/gsutil_test.go deleted file mode 100644 index 45c3cd814db..00000000000 --- a/pkg/skaffold/util/gsutil_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2020 The Skaffold Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "fmt" - "testing" - - "github.com/GoogleContainerTools/skaffold/v2/testutil" -) - -const ( - file = "source/file" - gcsFile = "gs://bucket/file" - folder = "source/" - gcsFolder = "gs://bucket/folder/" -) - -func TestCopy(t *testing.T) { - tests := []struct { - description string - src string - dst string - commands Command - recursive bool - shouldErr bool - }{ - { - description: "copy single file", - src: file, - dst: gcsFile, - commands: testutil.CmdRunOut(fmt.Sprintf("gsutil cp %s %s", file, gcsFile), "logs"), - }, - { - description: "copy recursively", - src: folder, - dst: gcsFolder, - commands: testutil.CmdRunOut(fmt.Sprintf("gsutil cp -r %s %s", folder, gcsFolder), "logs"), - recursive: true, - }, - { - description: "copy failed", - src: file, - dst: gcsFile, - commands: testutil.CmdRunOutErr(fmt.Sprintf("gsutil cp %s %s", file, gcsFile), "logs", fmt.Errorf("file not found")), - shouldErr: true, - }, - } - for _, test := range tests { - testutil.Run(t, test.description, func(t *testutil.T) { - t.Override(&DefaultExecCommand, test.commands) - - gcs := Gsutil{} - err := gcs.Copy(context.Background(), test.src, test.dst, test.recursive) - - t.CheckError(test.shouldErr, err) - }) - } -}