diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts index f183bcfd337..620832518c1 100644 --- a/core/src/plugins/kubernetes/config.ts +++ b/core/src/plugins/kubernetes/config.ts @@ -200,10 +200,17 @@ export interface NamespaceConfig { labels?: StringMap } +export interface ClusterBuildkitCacheConfig { + type: "registry" + mode: "min" | "max" | "auto" + tag: string + export: boolean +} + export interface KubernetesConfig extends BaseProviderConfig { buildMode: ContainerBuildMode clusterBuildkit?: { - overrideMultiStageCacheSupport?: boolean + cache: ClusterBuildkitCacheConfig[] rootless?: boolean nodeSelector?: StringMap } @@ -448,6 +455,46 @@ const tlsCertificateSchema = () => .example("cert-manager"), }) +const buildkitCacheConfigurationSchema = () => + joi.object().keys({ + type: joi + .string() + .allow("registry") + .required() + .description( + dedent` + See also the [buildkit registry cache documentation](https://github.com/moby/buildkit#registry-push-image-and-cache-separately) + ` + ), + mode: joi + .string() + .allow("auto", "min", "max") + .default("auto") + .description( + dedent` + See also the [buildkit export cache documentation](https://github.com/moby/buildkit#export-cache) + ` + ), + tag: joi + .string() + .default("_buildcache") + .description( + dedent` + This is the tag name for the registry build cache. Default is \`_buildcache\` + + **NOTE**: tag can only be used together with the \`registry\` cache type + ` + ), + export: joi + .boolean() + .default(true) + .description( + dedent` + If this is false, only import cache + ` + ), + }) + export const kubernetesConfigBase = () => providerConfigBaseSchema().keys({ buildMode: joi @@ -466,11 +513,16 @@ export const kubernetesConfigBase = () => clusterBuildkit: joi .object() .keys({ - overrideMultiStageCacheSupport: joi - .boolean() - .default(null) + cache: joi + .array() + .items(buildkitCacheConfigurationSchema()) + // TODO: fix default value, to include defaults automatically + .default([{ type: "registry", mode: "auto", tag: "_buildcache", export: true }]) .description( dedent` + + TODO!!!! + Enable the multi-stage cache (\`mode=max\`) buildkit option mode for builds using cluster-buildkit. Some registries are known not to support the cache manifests needed for the \`mode=max\` option, so @@ -510,7 +562,7 @@ export const kubernetesConfigBase = () => .example({ disktype: "ssd" }) .default(() => ({})), }) - .default(() => {}) + .default(() => ({})) .description("Configuration options for the `cluster-buildkit` build mode."), clusterDocker: joi .object() @@ -526,7 +578,7 @@ export const kubernetesConfigBase = () => ) .meta({ deprecated: true }), }) - .default(() => {}) + .default(() => ({})) .description("Configuration options for the `cluster-docker` build mode.") .meta({ deprecated: "The cluster-docker build mode has been deprecated." }), jib: joi diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts index e4cdd4c8d47..4b90c890389 100644 --- a/core/src/plugins/kubernetes/container/build/buildkit.ts +++ b/core/src/plugins/kubernetes/container/build/buildkit.ts @@ -15,7 +15,7 @@ import { KubeApi } from "../../api" import { KubernetesDeployment } from "../../types" import { LogEntry } from "../../../../logger/log-entry" import { waitForResources, compareDeployedResources } from "../../status/status" -import { KubernetesProvider, KubernetesPluginContext } from "../../config" +import { KubernetesProvider, KubernetesPluginContext, KubernetesConfig, ClusterBuildkitCacheConfig } from "../../config" import { PluginContext } from "../../../../plugin-context" import { BuildStatusHandler, @@ -71,31 +71,6 @@ export const getBuildkitBuildStatus: BuildStatusHandler = async (params) => { }) } -export const getMultiStageCacheSupport = (log: LogEntry, provider: KubernetesProvider, deploymentImageName: string) => { - const override = provider.config.clusterBuildkit?.overrideMultiStageCacheSupport - if (override !== null && override !== undefined) { - log.silly(`clusterBuildkit.overrideMultiStageCacheSupport is set to ${override}`) - return override - } - - // Detect AWS ECR - if (deploymentImageName.includes(".dkr.ecr.")) { - log.silly(`detected AWS ECR (=> not using mode=max)`) - return false - } - - // Detect gcr.io - if (deploymentImageName.includes("gcr.io")) { - log.silly(`detected Google Container Registry (=> not using mode=max)`) - return false - } - - log.silly("using mode=max") - - // Default to true for all others - return true -} - export const buildkitBuildHandler: BuildHandler = async (params) => { const { ctx, module, log } = params const provider = ctx.provider @@ -115,8 +90,6 @@ export const buildkitBuildHandler: BuildHandler = async (params) => { const deploymentImageId = module.outputs["deployment-image-id"] const dockerfile = module.spec.dockerfile || "Dockerfile" - const useModeMax = getMultiStageCacheSupport(log, provider, deploymentImageName) - const { contextPath } = await syncToBuildSync({ ...params, ctx: ctx as KubernetesPluginContext, @@ -140,29 +113,10 @@ export const buildkitBuildHandler: BuildHandler = async (params) => { statusLine.setState(renderOutputStream(line.toString())) }) - const cacheTag = "_buildcache" - - let outputSpec: string - let exportSpec: string - if (useModeMax) { - outputSpec = `type=image,"name=${deploymentImageId}",push=true` - exportSpec = `type=registry,mode=max,ref=${deploymentImageName}:${cacheTag}` - - if (usingInClusterRegistry(provider)) { - // The in-cluster registry is not exposed, so we don't configure TLS on it. - - // TODO(steffen): TEST THIS CASE - exportSpec += ",registry.insecure=true" - } - } else { - // for inline - outputSpec = `type=image,"name=${deploymentImageId},${deploymentImageName}:${cacheTag}",push=true` - exportSpec = "type=inline" - } - + let registryExtraSpec: string = "" if (usingInClusterRegistry(provider)) { // The in-cluster registry is not exposed, so we don't configure TLS on it. - outputSpec += ",registry.insecure=true" + registryExtraSpec = ",registry.insecure=true" } const command = [ @@ -176,12 +130,9 @@ export const buildkitBuildHandler: BuildHandler = async (params) => { "--opt", "filename=" + dockerfile, "--output", - outputSpec, - "--export-cache", - exportSpec, - "--import-cache", - `type=registry,ref=${deploymentImageName}:${cacheTag}`, - ...getBuildkitFlags(module), + `type=image,"name=${deploymentImageId}",push=true${registryExtraSpec}`, + ...getBuildkitCacheFlags(provider.config.clusterBuildkit!.cache, deploymentImageName, registryExtraSpec), + ...getBuildkitModuleFlags(module), ] // Execute the build @@ -280,7 +231,7 @@ export async function ensureBuildkit({ }) } -export function getBuildkitFlags(module: ContainerModule) { +export function getBuildkitModuleFlags(module: ContainerModule) { const args: string[] = [] for (const arg of getDockerBuildArgs(module)) { @@ -296,6 +247,56 @@ export function getBuildkitFlags(module: ContainerModule) { return args } +export function getBuildkitCacheFlags( + cacheConfig: ClusterBuildkitCacheConfig[], + deploymentImageName: string, + registryExtraSpec: string +) { + const args: string[] = [] + + // add import options + for (const cache of cacheConfig) { + args.push("--import-cache", `type=registry,ref=${deploymentImageName}:${cache.tag}${registryExtraSpec}`) + } + + // add export options + for (const cache of cacheConfig) { + if (!cache.export) { + continue + } + + const cacheMode = getSupportedCacheMode(cache, deploymentImageName) + args.push( + "--export-cache", + `type=registry,ref=${deploymentImageName}:${cache.tag},mode=${cacheMode}${registryExtraSpec}` + ) + } + + return args +} + +export const getSupportedCacheMode = ( + cache: ClusterBuildkitCacheConfig, + deploymentImageName: string +): ClusterBuildkitCacheConfig["mode"] => { + if (cache.mode !== "auto") { + return cache.mode + } + + // Detect AWS ECR + if (deploymentImageName.includes(".dkr.ecr.")) { + return "min" + } + + // Detect gcr.io + if (deploymentImageName.includes("gcr.io")) { + return "min" + } + + // Default to true for all others + return "max" +} + export function getBuildkitDeployment( provider: KubernetesProvider, authSecretName: string, @@ -356,6 +357,10 @@ export function getBuildkitDeployment( name: buildSyncVolumeName, mountPath: "/garden-build", }, + { + name: "garden-cache", + mountPath: "/tmp/garden-cache", + }, ], env: [ { @@ -385,6 +390,10 @@ export function getBuildkitDeployment( name: buildSyncVolumeName, emptyDir: {}, }, + { + name: "garden-cache", + emptyDir: {}, + }, ], tolerations: [builderToleration], }, diff --git a/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts b/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts index 572fb4ddd06..7383e751fd4 100644 --- a/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts +++ b/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts @@ -82,7 +82,7 @@ grouped("cluster-buildkit").describe("ensureBuildkit", () => { const nodeSelector = { "kubernetes.io/os": "linux" } - provider.config.clusterBuildkit = { nodeSelector } + provider.config.clusterBuildkit = { nodeSelector, cache: [] } await ensureBuildkit({ ctx, @@ -152,7 +152,7 @@ grouped("cluster-buildkit").describe("ensureBuildkit", () => { await api.apps.deleteNamespacedDeployment(buildkitDeploymentName, namespace) } catch {} - provider.config.clusterBuildkit = { rootless: true } + provider.config.clusterBuildkit = { rootless: true, cache: [] } await ensureBuildkit({ ctx, @@ -176,7 +176,7 @@ grouped("cluster-buildkit").describe("ensureBuildkit", () => { namespace, }) - provider.config.clusterBuildkit = { rootless: true } + provider.config.clusterBuildkit = { rootless: true, cache: [] } const { updated } = await ensureBuildkit({ ctx, diff --git a/core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts b/core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts index a4568e51be7..278dd52cde3 100644 --- a/core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts +++ b/core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts @@ -7,10 +7,10 @@ */ import { expect } from "chai" -import { getBuildkitFlags } from "../../../../../../../src/plugins/kubernetes/container/build/buildkit" +import { getBuildkitModuleFlags } from "../../../../../../../src/plugins/kubernetes/container/build/buildkit" import { getDataDir, makeTestGarden } from "../../../../../../helpers" -describe("getBuildkitFlags", () => { +describe("getBuildkitModuleFlags", () => { it("should correctly format the build target option", async () => { const projectRoot = getDataDir("test-project-container") const garden = await makeTestGarden(projectRoot) @@ -19,7 +19,7 @@ describe("getBuildkitFlags", () => { module.spec.build.targetImage = "foo" - const flags = getBuildkitFlags(module) + const flags = getBuildkitModuleFlags(module) expect(flags).to.eql([ "--opt",