From 507a6f1753c9feba3aa572283dbd14dd340bf95c Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Mon, 25 Dec 2023 18:58:56 +0300 Subject: [PATCH 1/4] Flexible image pull secret reference Co-authored-by: pat-s --- .../22-backends/40-kubernetes.md | 6 ++++ pipeline/backend/kubernetes/flags.go | 6 ++++ pipeline/backend/kubernetes/kubernetes.go | 28 ++++++++++--------- pipeline/backend/kubernetes/pod.go | 27 ++++++++++++++---- pipeline/backend/kubernetes/pod_test.go | 14 ++++------ 5 files changed, 55 insertions(+), 26 deletions(-) diff --git a/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md b/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md index 9ab79b7e6f2..234e52231e0 100644 --- a/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md +++ b/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md @@ -40,6 +40,12 @@ Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"exampl Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`. +### `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES` +> Default: `regcred` + +By default, in order to pull images from private repositories Kubernetes backend uses a secret named "regcred". +You can override it and also set many secret names. + ## Job specific configuration ### Resources diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 6bb7ec1aa87..4f55e9ec15c 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -74,4 +74,10 @@ var Flags = []cli.Flag{ Usage: "duration to wait before retrying to connect to the server", Value: time.Second * 2, }, + &cli.StringSliceFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES"}, + Name: "backend-k8s-pod-image-pull-secret-names", + Usage: "backend k8s pull secret names for private registries", + Value: cli.NewStringSlice("regcred"), + }, } diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index e8fb6820da3..d53bfec589e 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -54,13 +54,14 @@ type kube struct { } type config struct { - Namespace string - StorageClass string - VolumeSize string - StorageRwx bool - PodLabels map[string]string - PodAnnotations map[string]string - SecurityContext SecurityContextConfig + Namespace string + StorageClass string + VolumeSize string + StorageRwx bool + PodLabels map[string]string + PodAnnotations map[string]string + ImagePullSecretNames []string + SecurityContext SecurityContextConfig } type SecurityContextConfig struct { RunAsNonRoot bool @@ -80,12 +81,13 @@ func configFromCliContext(ctx context.Context) (*config, error) { if ctx != nil { if c, ok := ctx.Value(types.CliContext).(*cli.Context); ok { config := config{ - Namespace: c.String("backend-k8s-namespace"), - StorageClass: c.String("backend-k8s-storage-class"), - VolumeSize: c.String("backend-k8s-volume-size"), - StorageRwx: c.Bool("backend-k8s-storage-rwx"), - PodLabels: make(map[string]string), // just init empty map to prevent nil panic - PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic + Namespace: c.String("backend-k8s-namespace"), + StorageClass: c.String("backend-k8s-storage-class"), + VolumeSize: c.String("backend-k8s-volume-size"), + StorageRwx: c.Bool("backend-k8s-storage-rwx"), + PodLabels: make(map[string]string), // just init empty map to prevent nil panic + PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic + ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), SecurityContext: SecurityContextConfig{ RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), }, diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index d0adb5cc367..eb85f9f46e0 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -36,7 +36,7 @@ const ( func mkPod(namespace, name, image, workDir, goos, serviceAccountName string, pool, privileged bool, - commands, vols []string, + commands, vols, pullSecretNames []string, labels, annotations, env, nodeSelector map[string]string, extraHosts []types.HostAlias, tolerations []types.Toleration, resources types.Resources, securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig, @@ -45,7 +45,8 @@ func mkPod(namespace, name, image, workDir, goos, serviceAccountName string, meta := podMeta(name, namespace, labels, annotations) - spec, err := podSpec(serviceAccountName, vols, env, nodeSelector, extraHosts, tolerations, securityContext, securityContextConfig) + spec, err := podSpec(serviceAccountName, vols, pullSecretNames, env, nodeSelector, extraHosts, tolerations, + securityContext, securityContextConfig) if err != nil { return nil, err } @@ -85,7 +86,7 @@ func podMeta(name, namespace string, labels, annotations map[string]string) meta return meta } -func podSpec(serviceAccountName string, vols []string, env, backendNodeSelector map[string]string, +func podSpec(serviceAccountName string, vols, pullSecretNames []string, env, backendNodeSelector map[string]string, extraHosts []types.HostAlias, backendTolerations []types.Toleration, securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig, ) (v1.PodSpec, error) { @@ -93,7 +94,7 @@ func podSpec(serviceAccountName string, vols []string, env, backendNodeSelector spec := v1.PodSpec{ RestartPolicy: v1.RestartPolicyNever, ServiceAccountName: serviceAccountName, - ImagePullSecrets: []v1.LocalObjectReference{{Name: "regcred"}}, + ImagePullSecrets: imagePullSecretsReferences(pullSecretNames), } spec.HostAliases = hostAliases(extraHosts) @@ -211,6 +212,22 @@ func hostAlias(extraHost types.HostAlias) v1.HostAlias { } } +func imagePullSecretsReferences(imagePullSecretNames []string) []v1.LocalObjectReference { + log.Trace().Msgf("Using the image pull secrets: %v", imagePullSecretNames) + + secretReferences := make([]v1.LocalObjectReference, len(imagePullSecretNames)) + for i, imagePullSecretName := range imagePullSecretNames { + secretReferences[i] = imagePullSecretsReference(imagePullSecretName) + } + return secretReferences +} + +func imagePullSecretsReference(imagePullSecretName string) v1.LocalObjectReference { + return v1.LocalObjectReference{ + Name: imagePullSecretName, + } +} + func resourceRequirements(resources types.Resources) (v1.ResourceRequirements, error) { var err error requirements := v1.ResourceRequirements{} @@ -357,7 +374,7 @@ func startPod(ctx context.Context, engine *kube, step *types.Step) (*v1.Pod, err pod, err := mkPod(engine.config.Namespace, podName, step.Image, step.WorkingDir, engine.goos, step.BackendOptions.Kubernetes.ServiceAccountName, step.Pull, step.Privileged, - step.Commands, step.Volumes, + step.Commands, step.Volumes, engine.config.ImagePullSecretNames, engine.config.PodLabels, engine.config.PodAnnotations, step.Environment, step.BackendOptions.Kubernetes.NodeSelector, step.ExtraHosts, step.BackendOptions.Kubernetes.Tolerations, step.BackendOptions.Kubernetes.Resources, step.BackendOptions.Kubernetes.SecurityContext, engine.config.SecurityContext) if err != nil { diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index b05b2451e44..fe33cb63075 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -96,19 +96,14 @@ func TestTinyPod(t *testing.T) { ] } ], - "restartPolicy": "Never", - "imagePullSecrets": [ - { - "name": "regcred" - } - ] + "restartPolicy": "Never" }, "status": {} }` pod, err := mkPod("woodpecker", "wp-01he8bebctabr3kgk0qj36d2me-0", "gradle:8.4.0-jdk21", "/woodpecker/src", "linux/amd64", "", false, false, - []string{"gradle build"}, []string{"workspace:/woodpecker/src"}, + []string{"gradle build"}, []string{"workspace:/woodpecker/src"}, nil, nil, nil, map[string]string{"CI": "woodpecker"}, nil, nil, nil, types.Resources{Requests: nil, Limits: nil}, nil, SecurityContextConfig{}, @@ -213,6 +208,9 @@ func TestFullPod(t *testing.T) { "imagePullSecrets": [ { "name": "regcred" + }, + { + "name": "another-pull-secret" } ], "tolerations": [ @@ -246,7 +244,7 @@ func TestFullPod(t *testing.T) { } pod, err := mkPod("woodpecker", "wp-01he8bebctabr3kgk0qj36d2me-0", "meltwater/drone-cache", "/woodpecker/src", "linux/amd64", "wp-svc-acc", true, true, - []string{"go get", "go test"}, []string{"woodpecker-cache:/woodpecker/src/cache"}, + []string{"go get", "go test"}, []string{"woodpecker-cache:/woodpecker/src/cache"}, []string{"regcred", "another-pull-secret"}, map[string]string{"app": "test"}, map[string]string{"apparmor.security": "runtime/default"}, map[string]string{"CGO": "0"}, map[string]string{"storage": "ssd"}, hostAliases, []types.Toleration{{Key: "net-port", Value: "100Mbit", Effect: types.TaintEffectNoSchedule}}, types.Resources{Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"}, Limits: map[string]string{"memory": "256Mi", "cpu": "2"}}, From 87f7897333624dafda994d24c60fdb81c00c6809 Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:37:00 +0300 Subject: [PATCH 2/4] PR notes --- .../30-administration/22-backends/40-kubernetes.md | 6 ------ .../30-administration/22-backends/40-kubernetes.md | 4 ++++ docs/versioned_docs/version-2.1/91-migrations.md | 1 + pipeline/backend/kubernetes/flags.go | 1 - pipeline/backend/kubernetes/kubernetes.go | 5 +++++ 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md b/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md index 234e52231e0..9ab79b7e6f2 100644 --- a/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md +++ b/docs/versioned_docs/version-1.0/30-administration/22-backends/40-kubernetes.md @@ -40,12 +40,6 @@ Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"exampl Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`. -### `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES` -> Default: `regcred` - -By default, in order to pull images from private repositories Kubernetes backend uses a secret named "regcred". -You can override it and also set many secret names. - ## Job specific configuration ### Resources diff --git a/docs/versioned_docs/version-2.1/30-administration/22-backends/40-kubernetes.md b/docs/versioned_docs/version-2.1/30-administration/22-backends/40-kubernetes.md index abd1b5b2869..7efb478c840 100644 --- a/docs/versioned_docs/version-2.1/30-administration/22-backends/40-kubernetes.md +++ b/docs/versioned_docs/version-2.1/30-administration/22-backends/40-kubernetes.md @@ -46,6 +46,10 @@ agent: Determines if containers must be required to run as non-root users. +- `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES` (default: empty) + + Secret names to pull images from private repositories. + ## Job specific configuration ### Resources diff --git a/docs/versioned_docs/version-2.1/91-migrations.md b/docs/versioned_docs/version-2.1/91-migrations.md index 328d87a6255..f98ea4c5b03 100644 --- a/docs/versioned_docs/version-2.1/91-migrations.md +++ b/docs/versioned_docs/version-2.1/91-migrations.md @@ -7,6 +7,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Deprecated `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](./20-usage/20-workflow-syntax.md#depends_on) to learn how to set dependencies) - Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead - Pipelines without a config file will now be skipped instead of failing +- Removed implicitly defined `regcred` image pull secret. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`. ## 2.0.0 diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 4f55e9ec15c..963a9b1c524 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -78,6 +78,5 @@ var Flags = []cli.Flag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES"}, Name: "backend-k8s-pod-image-pull-secret-names", Usage: "backend k8s pull secret names for private registries", - Value: cli.NewStringSlice("regcred"), }, } diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index d53bfec589e..30c182a4558 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -92,6 +92,11 @@ func configFromCliContext(ctx context.Context) (*config, error) { RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), }, } + // TODO: remove in next major + if len(config.ImagePullSecretNames) == 0 { + log.Warn().Msgf("WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES is not set, using 'regcred'. It will be removed in next major. Set it explicitly before.") + config.ImagePullSecretNames = []string{"regcred"} + } // Unmarshal label and annotation settings here to ensure they're valid on startup if labels := c.String("backend-k8s-pod-labels"); labels != "" { if err := yaml.Unmarshal([]byte(labels), &config.PodLabels); err != nil { From 1c65abd67774c3a687a9f88f62bf1a7aa08e64c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:50:18 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks [CI SKIP] for more information, see https://pre-commit.ci --- docs/versioned_docs/version-2.1/91-migrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioned_docs/version-2.1/91-migrations.md b/docs/versioned_docs/version-2.1/91-migrations.md index f98ea4c5b03..1980059abcb 100644 --- a/docs/versioned_docs/version-2.1/91-migrations.md +++ b/docs/versioned_docs/version-2.1/91-migrations.md @@ -7,7 +7,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Deprecated `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](./20-usage/20-workflow-syntax.md#depends_on) to learn how to set dependencies) - Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead - Pipelines without a config file will now be skipped instead of failing -- Removed implicitly defined `regcred` image pull secret. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`. +- Removed implicitly defined `regcred` image pull secret. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`. ## 2.0.0 From e131474c12a2b3722a11107c48ad50561ddeb0b2 Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Tue, 2 Jan 2024 22:53:18 +0300 Subject: [PATCH 4/4] PR notes --- docs/versioned_docs/version-2.1/91-migrations.md | 2 +- pipeline/backend/kubernetes/flags.go | 1 + pipeline/backend/kubernetes/kubernetes.go | 5 ++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/versioned_docs/version-2.1/91-migrations.md b/docs/versioned_docs/version-2.1/91-migrations.md index 1980059abcb..921580e022e 100644 --- a/docs/versioned_docs/version-2.1/91-migrations.md +++ b/docs/versioned_docs/version-2.1/91-migrations.md @@ -7,7 +7,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Deprecated `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](./20-usage/20-workflow-syntax.md#depends_on) to learn how to set dependencies) - Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead - Pipelines without a config file will now be skipped instead of failing -- Removed implicitly defined `regcred` image pull secret. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`. +- Deprecated implicitly defined `regcred` image pull secret. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`. ## 2.0.0 diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 963a9b1c524..4f55e9ec15c 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -78,5 +78,6 @@ var Flags = []cli.Flag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES"}, Name: "backend-k8s-pod-image-pull-secret-names", Usage: "backend k8s pull secret names for private registries", + Value: cli.NewStringSlice("regcred"), }, } diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 30c182a4558..4620dcfd204 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -93,9 +93,8 @@ func configFromCliContext(ctx context.Context) (*config, error) { }, } // TODO: remove in next major - if len(config.ImagePullSecretNames) == 0 { - log.Warn().Msgf("WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES is not set, using 'regcred'. It will be removed in next major. Set it explicitly before.") - config.ImagePullSecretNames = []string{"regcred"} + if len(config.ImagePullSecretNames) == 1 && config.ImagePullSecretNames[0] == "regcred" { + log.Warn().Msg("WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES is set to the default ('regcred'). It will default to empty in Woodpecker 3.0. Set it explicitly before then.") } // Unmarshal label and annotation settings here to ensure they're valid on startup if labels := c.String("backend-k8s-pod-labels"); labels != "" {