Skip to content

Commit

Permalink
feat: flexible cache configuration
Browse files Browse the repository at this point in the history 
Co-authored-by: Tim <tim@garden.io>
  • Loading branch information
@stefreak
stefreak and Tim committed Sep 21, 2022
1 parent 70be70b commit f3c486d
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 68 deletions.
64 changes: 58 additions & 6 deletions core/src/plugins/kubernetes/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,10 +200,17 @@ export interface NamespaceConfig {
labels?: StringMap
}

export interface ClusterBuildkitCacheConfig {
type: "registry"
mode: "min" | "max" | "auto"
tag: string
export: boolean
}

export interface KubernetesConfig extends BaseProviderConfig {
buildMode: ContainerBuildMode
clusterBuildkit?: {
overrideMultiStageCacheSupport?: boolean
cache: ClusterBuildkitCacheConfig[]
rootless?: boolean
nodeSelector?: StringMap
}
Expand Down Expand Up @@ -448,6 +455,46 @@ const tlsCertificateSchema = () =>
.example("cert-manager"),
})

const buildkitCacheConfigurationSchema = () =>
joi.object().keys({
type: joi
.string()
.allow("registry")
.required()
.description(
dedent`
See also the [buildkit registry cache documentation](https://github.com/moby/buildkit#registry-push-image-and-cache-separately)
`
),
mode: joi
.string()
.allow("auto", "min", "max")
.default("auto")
.description(
dedent`
See also the [buildkit export cache documentation](https://github.com/moby/buildkit#export-cache)
`
),
tag: joi
.string()
.default("_buildcache")
.description(
dedent`
This is the tag name for the registry build cache. Default is \`_buildcache\`
**NOTE**: tag can only be used together with the \`registry\` cache type
`
),
export: joi
.boolean()
.default(true)
.description(
dedent`
If this is false, only import cache
`
),
})

export const kubernetesConfigBase = () =>
providerConfigBaseSchema().keys({
buildMode: joi
Expand All @@ -466,11 +513,16 @@ export const kubernetesConfigBase = () =>
clusterBuildkit: joi
.object()
.keys({
overrideMultiStageCacheSupport: joi
.boolean()
.default(null)
cache: joi
.array()
.items(buildkitCacheConfigurationSchema())
// TODO: fix default value, to include defaults automatically
.default([{ type: "registry", mode: "auto", tag: "_buildcache", export: true }])
.description(
dedent`
TODO!!!!
Enable the multi-stage cache (\`mode=max\`) buildkit option mode for builds using cluster-buildkit.
Some registries are known not to support the cache manifests needed for the \`mode=max\` option, so
Expand Down Expand Up @@ -510,7 +562,7 @@ export const kubernetesConfigBase = () =>
.example({ disktype: "ssd" })
.default(() => ({})),
})
.default(() => {})
.default(() => ({}))
.description("Configuration options for the `cluster-buildkit` build mode."),
clusterDocker: joi
.object()
Expand All @@ -526,7 +578,7 @@ export const kubernetesConfigBase = () =>
)
.meta({ deprecated: true }),
})
.default(() => {})
.default(() => ({}))
.description("Configuration options for the `cluster-docker` build mode.")
.meta({ deprecated: "The cluster-docker build mode has been deprecated." }),
jib: joi
Expand Down
121 changes: 65 additions & 56 deletions core/src/plugins/kubernetes/container/build/buildkit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { KubeApi } from "../../api"
import { KubernetesDeployment } from "../../types"
import { LogEntry } from "../../../../logger/log-entry"
import { waitForResources, compareDeployedResources } from "../../status/status"
import { KubernetesProvider, KubernetesPluginContext } from "../../config"
import { KubernetesProvider, KubernetesPluginContext, KubernetesConfig, ClusterBuildkitCacheConfig } from "../../config"
import { PluginContext } from "../../../../plugin-context"
import {
BuildStatusHandler,
Expand Down Expand Up @@ -71,31 +71,6 @@ export const getBuildkitBuildStatus: BuildStatusHandler = async (params) => {
})
}

export const getMultiStageCacheSupport = (log: LogEntry, provider: KubernetesProvider, deploymentImageName: string) => {
const override = provider.config.clusterBuildkit?.overrideMultiStageCacheSupport
if (override !== null && override !== undefined) {
log.silly(`clusterBuildkit.overrideMultiStageCacheSupport is set to ${override}`)
return override
}

// Detect AWS ECR
if (deploymentImageName.includes(".dkr.ecr.")) {
log.silly(`detected AWS ECR (=> not using mode=max)`)
return false
}

// Detect gcr.io
if (deploymentImageName.includes("gcr.io")) {
log.silly(`detected Google Container Registry (=> not using mode=max)`)
return false
}

log.silly("using mode=max")

// Default to true for all others
return true
}

export const buildkitBuildHandler: BuildHandler = async (params) => {
const { ctx, module, log } = params
const provider = <KubernetesProvider>ctx.provider
Expand All @@ -115,8 +90,6 @@ export const buildkitBuildHandler: BuildHandler = async (params) => {
const deploymentImageId = module.outputs["deployment-image-id"]
const dockerfile = module.spec.dockerfile || "Dockerfile"

const useModeMax = getMultiStageCacheSupport(log, provider, deploymentImageName)

const { contextPath } = await syncToBuildSync({
...params,
ctx: ctx as KubernetesPluginContext,
Expand All @@ -140,29 +113,10 @@ export const buildkitBuildHandler: BuildHandler = async (params) => {
statusLine.setState(renderOutputStream(line.toString()))
})

const cacheTag = "_buildcache"

let outputSpec: string
let exportSpec: string
if (useModeMax) {
outputSpec = `type=image,"name=${deploymentImageId}",push=true`
exportSpec = `type=registry,mode=max,ref=${deploymentImageName}:${cacheTag}`

if (usingInClusterRegistry(provider)) {
// The in-cluster registry is not exposed, so we don't configure TLS on it.

// TODO(steffen): TEST THIS CASE
exportSpec += ",registry.insecure=true"
}
} else {
// for inline
outputSpec = `type=image,"name=${deploymentImageId},${deploymentImageName}:${cacheTag}",push=true`
exportSpec = "type=inline"
}

let registryExtraSpec: string = ""
if (usingInClusterRegistry(provider)) {
// The in-cluster registry is not exposed, so we don't configure TLS on it.
outputSpec += ",registry.insecure=true"
registryExtraSpec = ",registry.insecure=true"
}

const command = [
Expand All @@ -176,12 +130,9 @@ export const buildkitBuildHandler: BuildHandler = async (params) => {
"--opt",
"filename=" + dockerfile,
"--output",
outputSpec,
"--export-cache",
exportSpec,
"--import-cache",
`type=registry,ref=${deploymentImageName}:${cacheTag}`,
...getBuildkitFlags(module),
`type=image,"name=${deploymentImageId}",push=true${registryExtraSpec}`,
...getBuildkitCacheFlags(provider.config.clusterBuildkit!.cache, deploymentImageName, registryExtraSpec),
...getBuildkitModuleFlags(module),
]

// Execute the build
Expand Down Expand Up @@ -280,7 +231,7 @@ export async function ensureBuildkit({
})
}

export function getBuildkitFlags(module: ContainerModule) {
export function getBuildkitModuleFlags(module: ContainerModule) {
const args: string[] = []

for (const arg of getDockerBuildArgs(module)) {
Expand All @@ -296,6 +247,56 @@ export function getBuildkitFlags(module: ContainerModule) {
return args
}

export function getBuildkitCacheFlags(
cacheConfig: ClusterBuildkitCacheConfig[],
deploymentImageName: string,
registryExtraSpec: string
) {
const args: string[] = []

// add import options
for (const cache of cacheConfig) {
args.push("--import-cache", `type=registry,ref=${deploymentImageName}:${cache.tag}${registryExtraSpec}`)
}

// add export options
for (const cache of cacheConfig) {
if (!cache.export) {
continue
}

const cacheMode = getSupportedCacheMode(cache, deploymentImageName)
args.push(
"--export-cache",
`type=registry,ref=${deploymentImageName}:${cache.tag},mode=${cacheMode}${registryExtraSpec}`
)
}

return args
}

export const getSupportedCacheMode = (
cache: ClusterBuildkitCacheConfig,
deploymentImageName: string
): ClusterBuildkitCacheConfig["mode"] => {
if (cache.mode !== "auto") {
return cache.mode
}

// Detect AWS ECR
if (deploymentImageName.includes(".dkr.ecr.")) {
return "min"
}

// Detect gcr.io
if (deploymentImageName.includes("gcr.io")) {
return "min"
}

// Default to true for all others
return "max"
}

export function getBuildkitDeployment(
provider: KubernetesProvider,
authSecretName: string,
Expand Down Expand Up @@ -356,6 +357,10 @@ export function getBuildkitDeployment(
name: buildSyncVolumeName,
mountPath: "/garden-build",
},
{
name: "garden-cache",
mountPath: "/tmp/garden-cache",
},
],
env: [
{
Expand Down Expand Up @@ -385,6 +390,10 @@ export function getBuildkitDeployment(
name: buildSyncVolumeName,
emptyDir: {},
},
{
name: "garden-cache",
emptyDir: {},
},
],
tolerations: [builderToleration],
},
Expand Down
6 changes: 3 additions & 3 deletions core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ grouped("cluster-buildkit").describe("ensureBuildkit", () => {

const nodeSelector = { "kubernetes.io/os": "linux" }

provider.config.clusterBuildkit = { nodeSelector }
provider.config.clusterBuildkit = { nodeSelector, cache: [] }

await ensureBuildkit({
ctx,
Expand Down Expand Up @@ -152,7 +152,7 @@ grouped("cluster-buildkit").describe("ensureBuildkit", () => {
await api.apps.deleteNamespacedDeployment(buildkitDeploymentName, namespace)
} catch {}

provider.config.clusterBuildkit = { rootless: true }
provider.config.clusterBuildkit = { rootless: true, cache: [] }

await ensureBuildkit({
ctx,
Expand All @@ -176,7 +176,7 @@ grouped("cluster-buildkit").describe("ensureBuildkit", () => {
namespace,
})

provider.config.clusterBuildkit = { rootless: true }
provider.config.clusterBuildkit = { rootless: true, cache: [] }

const { updated } = await ensureBuildkit({
ctx,
Expand Down
6 changes: 3 additions & 3 deletions core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
*/

import { expect } from "chai"
import { getBuildkitFlags } from "../../../../../../../src/plugins/kubernetes/container/build/buildkit"
import { getBuildkitModuleFlags } from "../../../../../../../src/plugins/kubernetes/container/build/buildkit"
import { getDataDir, makeTestGarden } from "../../../../../../helpers"

describe("getBuildkitFlags", () => {
describe("getBuildkitModuleFlags", () => {
it("should correctly format the build target option", async () => {
const projectRoot = getDataDir("test-project-container")
const garden = await makeTestGarden(projectRoot)
Expand All @@ -19,7 +19,7 @@ describe("getBuildkitFlags", () => {

module.spec.build.targetImage = "foo"

const flags = getBuildkitFlags(module)
const flags = getBuildkitModuleFlags(module)

expect(flags).to.eql([
"--opt",
Expand Down

0 comments on commit f3c486d

Please sign in to comment.