From 64ca9a54d10da444ad0685a031f40ec6762b82c5 Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Sat, 13 Apr 2024 02:30:05 +0300 Subject: [PATCH 1/4] Added step's labels and annotations --- .../backend/kubernetes/backend_options.go | 2 + .../kubernetes/backend_options_test.go | 4 ++ pipeline/backend/kubernetes/pod.go | 61 +++++++++++++------ pipeline/backend/kubernetes/pod_test.go | 6 +- .../frontend/yaml/linter/schema/schema.json | 13 +++- 5 files changed, 65 insertions(+), 21 deletions(-) diff --git a/pipeline/backend/kubernetes/backend_options.go b/pipeline/backend/kubernetes/backend_options.go index 3b1109a2c51..a1d82067e98 100644 --- a/pipeline/backend/kubernetes/backend_options.go +++ b/pipeline/backend/kubernetes/backend_options.go @@ -11,6 +11,8 @@ type BackendOptions struct { Resources Resources `mapstructure:"resources"` RuntimeClassName *string `mapstructure:"runtimeClassName"` ServiceAccountName string `mapstructure:"serviceAccountName"` + Labels map[string]string `mapstructure:"labels"` + Annotations map[string]string `mapstructure:"annotations"` NodeSelector map[string]string `mapstructure:"nodeSelector"` Tolerations []Toleration `mapstructure:"tolerations"` SecurityContext *SecurityContext `mapstructure:"securityContext"` diff --git a/pipeline/backend/kubernetes/backend_options_test.go b/pipeline/backend/kubernetes/backend_options_test.go index f9c21a5fe05..3d5a73f3230 100644 --- a/pipeline/backend/kubernetes/backend_options_test.go +++ b/pipeline/backend/kubernetes/backend_options_test.go @@ -20,6 +20,8 @@ func Test_parseBackendOptions(t *testing.T) { "kubernetes": map[string]any{ "nodeSelector": map[string]string{"storage": "ssd"}, "serviceAccountName": "wp-svc-acc", + "labels": map[string]string{"app": "test"}, + "annotations": map[string]string{"apps.kubernetes.io/pod-index": "0"}, "tolerations": []map[string]any{ {"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule}, }, @@ -49,6 +51,8 @@ func Test_parseBackendOptions(t *testing.T) { assert.Equal(t, BackendOptions{ NodeSelector: map[string]string{"storage": "ssd"}, ServiceAccountName: "wp-svc-acc", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, Tolerations: []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}}, Resources: Resources{ Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"}, diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index 331efae92d9..aa0cf355d13 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -76,41 +76,66 @@ func podName(step *types.Step) (string, error) { func podMeta(step *types.Step, config *config, options BackendOptions, podName string) (metav1.ObjectMeta, error) { var err error meta := metav1.ObjectMeta{ - Name: podName, - Namespace: config.Namespace, + Name: podName, + Namespace: config.Namespace, + Annotations: podAnnotations(config, options, podName), } - meta.Labels = config.PodLabels - if meta.Labels == nil { - meta.Labels = make(map[string]string, 1) - } - meta.Labels[StepLabel], err = stepLabel(step) + meta.Labels, err = podLabels(step, config, options) if err != nil { return meta, err } + return meta, nil +} + +func podLabels(step *types.Step, config *config, options BackendOptions) (map[string]string, error) { + var err error + labels := make(map[string]string) + + if len(options.Labels) > 0 { + log.Trace().Msgf("using labels from the backend options: %v", options.Labels) + maps.Copy(labels, options.Labels) + } + if len(config.PodLabels) > 0 { + log.Trace().Msgf("using labels from the configuration: %v", config.PodLabels) + maps.Copy(labels, config.PodLabels) + } if step.Type == types.StepTypeService { - meta.Labels[ServiceLabel], _ = serviceName(step) + labels[ServiceLabel], _ = serviceName(step) } - - meta.Annotations = config.PodAnnotations - if meta.Annotations == nil { - meta.Annotations = make(map[string]string) + labels[StepLabel], err = stepLabel(step) + if err != nil { + return labels, err } + return labels, nil +} + +func stepLabel(step *types.Step) (string, error) { + return toDNSName(step.Name) +} + +func podAnnotations(config *config, options BackendOptions, podName string) map[string]string { + annotations := make(map[string]string) + + if len(options.Annotations) > 0 { + log.Trace().Msgf("using annotations from the backend options: %v", options.Annotations) + maps.Copy(annotations, options.Annotations) + } + if len(config.PodAnnotations) > 0 { + log.Trace().Msgf("using annotations from the configuration: %v", config.PodAnnotations) + maps.Copy(annotations, config.PodAnnotations) + } securityContext := options.SecurityContext if securityContext != nil { key, value := apparmorAnnotation(podName, securityContext.ApparmorProfile) if key != nil && value != nil { - meta.Annotations[*key] = *value + annotations[*key] = *value } } - return meta, nil -} - -func stepLabel(step *types.Step) (string, error) { - return toDNSName(step.Name) + return annotations } func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSpec, error) { diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index bf4f35a4195..412b30579f7 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -159,11 +159,13 @@ func TestFullPod(t *testing.T) { "creationTimestamp": null, "labels": { "app": "test", + "part-of": "woodpecker-ci", "step": "go-test" }, "annotations": { "apps.kubernetes.io/pod-index": "0", - "container.apparmor.security.beta.kubernetes.io/wp-01he8bebctabr3kgk0qj36d2me-0": "localhost/k8s-apparmor-example-deny-write" + "container.apparmor.security.beta.kubernetes.io/wp-01he8bebctabr3kgk0qj36d2me-0": "localhost/k8s-apparmor-example-deny-write", + "kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container" } }, "spec": { @@ -334,6 +336,8 @@ func TestFullPod(t *testing.T) { PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Labels: map[string]string{"part-of": "woodpecker-ci"}, + Annotations: map[string]string{"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"}, NodeSelector: map[string]string{"storage": "ssd"}, RuntimeClassName: &runtimeClass, ServiceAccountName: "wp-svc-acc", diff --git a/pipeline/frontend/yaml/linter/schema/schema.json b/pipeline/frontend/yaml/linter/schema/schema.json index 13d7af690ec..ee04a5a0913 100644 --- a/pipeline/frontend/yaml/linter/schema/schema.json +++ b/pipeline/frontend/yaml/linter/schema/schema.json @@ -691,8 +691,17 @@ "description": "Advanced options for the kubernetes agent backends", "type": "object", "properties": { - "resources": { - "$ref": "#/definitions/step_backend_kubernetes_resources" + "labels": { + "type": "object", + "additionalProperties": { + "type": ["boolean", "string", "number"] + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": ["boolean", "string", "number"] + } }, "securityContext": { "$ref": "#/definitions/step_backend_kubernetes_security_context" From 614d02f404d75bccaed67058d9d764e966d10bdd Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Sat, 13 Apr 2024 02:39:40 +0300 Subject: [PATCH 2/4] Added feature flags --- pipeline/backend/kubernetes/flags.go | 16 +++++++++-- pipeline/backend/kubernetes/kubernetes.go | 34 +++++++++++++---------- pipeline/backend/kubernetes/pod.go | 4 +-- pipeline/backend/kubernetes/pod_test.go | 12 ++++---- 4 files changed, 42 insertions(+), 24 deletions(-) diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 6d5fd9c92dd..347d0bd6459 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -46,15 +46,27 @@ var Flags = []cli.Flag{ &cli.StringFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"}, Name: "backend-k8s-pod-labels", - Usage: "backend k8s additional worker pod labels", + Usage: "backend k8s additional Agent-wide worker pod labels", Value: "", }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP"}, + Name: "backend-k8s-pod-labels-allow-from-step", + Usage: "whether to allow using labels from step's backend options", + Value: false, + }, &cli.StringFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"}, Name: "backend-k8s-pod-annotations", - Usage: "backend k8s additional worker pod annotations", + Usage: "backend k8s additional Agent-wide worker pod annotations", Value: "", }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"}, + Name: "backend-k8s-pod-annotations-allow-from-step", + Usage: "whether to allow using annotations from step's backend options", + Value: false, + }, &cli.BoolFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"}, Name: "backend-k8s-secctx-nonroot", diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 3567100a6db..5e97f12a784 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -55,14 +55,16 @@ type kube struct { } type config struct { - Namespace string - StorageClass string - VolumeSize string - StorageRwx bool - PodLabels map[string]string - PodAnnotations map[string]string - ImagePullSecretNames []string - SecurityContext SecurityContextConfig + Namespace string + StorageClass string + VolumeSize string + StorageRwx bool + PodLabels map[string]string + PodLabelsAllowFromStep bool + PodAnnotations map[string]string + PodAnnotationsAllowFromStep bool + ImagePullSecretNames []string + SecurityContext SecurityContextConfig } type SecurityContextConfig struct { RunAsNonRoot bool @@ -82,13 +84,15 @@ func configFromCliContext(ctx context.Context) (*config, error) { if ctx != nil { if c, ok := ctx.Value(types.CliContext).(*cli.Context); ok { config := config{ - Namespace: c.String("backend-k8s-namespace"), - StorageClass: c.String("backend-k8s-storage-class"), - VolumeSize: c.String("backend-k8s-volume-size"), - StorageRwx: c.Bool("backend-k8s-storage-rwx"), - PodLabels: make(map[string]string), // just init empty map to prevent nil panic - PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic - ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), + Namespace: c.String("backend-k8s-namespace"), + StorageClass: c.String("backend-k8s-storage-class"), + VolumeSize: c.String("backend-k8s-volume-size"), + StorageRwx: c.Bool("backend-k8s-storage-rwx"), + PodLabels: make(map[string]string), // just init empty map to prevent nil panic + PodLabelsAllowFromStep: c.Bool("backend-k8s-pod-labels-allow-from-step"), + PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic + PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"), + ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), SecurityContext: SecurityContextConfig{ RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), }, diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index aa0cf355d13..0ce5712ee69 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -93,7 +93,7 @@ func podLabels(step *types.Step, config *config, options BackendOptions) (map[st var err error labels := make(map[string]string) - if len(options.Labels) > 0 { + if config.PodLabelsAllowFromStep && len(options.Labels) > 0 { log.Trace().Msgf("using labels from the backend options: %v", options.Labels) maps.Copy(labels, options.Labels) } @@ -119,7 +119,7 @@ func stepLabel(step *types.Step) (string, error) { func podAnnotations(config *config, options BackendOptions, podName string) map[string]string { annotations := make(map[string]string) - if len(options.Annotations) > 0 { + if config.PodAnnotationsAllowFromStep && len(options.Annotations) > 0 { log.Trace().Msgf("using annotations from the backend options: %v", options.Annotations) maps.Copy(annotations, options.Annotations) } diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index 412b30579f7..ebd134fc153 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -330,11 +330,13 @@ func TestFullPod(t *testing.T) { ExtraHosts: hostAliases, Ports: ports, }, &config{ - Namespace: "woodpecker", - ImagePullSecretNames: []string{"regcred", "another-pull-secret"}, - PodLabels: map[string]string{"app": "test"}, - PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, - SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, + Namespace: "woodpecker", + ImagePullSecretNames: []string{"regcred", "another-pull-secret"}, + PodLabels: map[string]string{"app": "test"}, + PodLabelsAllowFromStep: true, + PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, + PodAnnotationsAllowFromStep: true, + SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ Labels: map[string]string{"part-of": "woodpecker-ci"}, Annotations: map[string]string{"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"}, From 93327cf1a415bfb9403129fa6c9e1ed2725cfaca Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Sun, 28 Apr 2024 16:47:44 +0300 Subject: [PATCH 3/4] Added warning --- pipeline/backend/kubernetes/pod.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index 0ce5712ee69..38183380c49 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -93,9 +93,13 @@ func podLabels(step *types.Step, config *config, options BackendOptions) (map[st var err error labels := make(map[string]string) - if config.PodLabelsAllowFromStep && len(options.Labels) > 0 { - log.Trace().Msgf("using labels from the backend options: %v", options.Labels) - maps.Copy(labels, options.Labels) + if len(options.Labels) > 0 { + if config.PodLabelsAllowFromStep { + log.Trace().Msgf("using labels from the backend options: %v", options.Labels) + maps.Copy(labels, options.Labels) + } else { + log.Debug().Msg("Pod labels were defined in backend options, but its using disallowed by instance configuration ") + } } if len(config.PodLabels) > 0 { log.Trace().Msgf("using labels from the configuration: %v", config.PodLabels) @@ -119,9 +123,13 @@ func stepLabel(step *types.Step) (string, error) { func podAnnotations(config *config, options BackendOptions, podName string) map[string]string { annotations := make(map[string]string) - if config.PodAnnotationsAllowFromStep && len(options.Annotations) > 0 { - log.Trace().Msgf("using annotations from the backend options: %v", options.Annotations) - maps.Copy(annotations, options.Annotations) + if len(options.Annotations) > 0 { + if config.PodAnnotationsAllowFromStep { + log.Trace().Msgf("using annotations from the backend options: %v", options.Annotations) + maps.Copy(annotations, options.Annotations) + } else { + log.Debug().Msg("Pod annotations were defined in backend options, but its using disallowed by instance configuration ") + } } if len(config.PodAnnotations) > 0 { log.Trace().Msgf("using annotations from the configuration: %v", config.PodAnnotations) From f41956eaa77e0710589ec842abda9f8eb3117e9b Mon Sep 17 00:00:00 2001 From: qwerty287 <80460567+qwerty287@users.noreply.github.com> Date: Sat, 11 May 2024 11:19:16 +0200 Subject: [PATCH 4/4] Update pod.go --- pipeline/backend/kubernetes/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index d712fc2c70d..2481e90c7d0 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -98,7 +98,7 @@ func podLabels(step *types.Step, config *config, options BackendOptions) (map[st log.Trace().Msgf("using labels from the backend options: %v", options.Labels) maps.Copy(labels, options.Labels) } else { - log.Debug().Msg("Pod labels were defined in backend options, but its using disallowed by instance configuration ") + log.Debug().Msg("Pod labels were defined in backend options, but its using disallowed by instance configuration") } } if len(config.PodLabels) > 0 {