From de77d834603d2b9c391c9037c56354901373730a Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Mon, 24 Oct 2022 14:15:05 -0500 Subject: [PATCH 01/13] add tolerations --- internal/k8s/builder/gateway.go | 1 + .../builder/testdata/static-mapping.deployment.golden.yaml | 5 +++++ internal/k8s/builder/testdata/static-mapping.yaml | 5 +++++ pkg/apis/v1alpha1/types.go | 3 +++ 4 files changed, 14 insertions(+) diff --git a/internal/k8s/builder/gateway.go b/internal/k8s/builder/gateway.go index 7503dbcdc..73c473f48 100644 --- a/internal/k8s/builder/gateway.go +++ b/internal/k8s/builder/gateway.go @@ -199,6 +199,7 @@ func (b *GatewayDeploymentBuilder) podSpec() corev1.PodSpec { }, }, NodeSelector: b.gwConfig.Spec.NodeSelector, + Tolerations: b.gwConfig.Spec.Tolerations, ServiceAccountName: orDefault(b.gwConfig.Spec.ConsulSpec.AuthSpec.Account, defaultServiceAccount), // the init container copies the binary into the // next envoy container so we can decouple the envoy diff --git a/internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml b/internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml index e173de3fc..dd5aed5a4 100644 --- a/internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml +++ b/internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml @@ -115,6 +115,11 @@ spec: nodeSelector: ingress-ready: "true" serviceAccountName: consul-api-gateway + tolerations: + - effect: NoSchedule + key: key1 + operator: Equal + value: value1 volumes: - emptyDir: {} name: bootstrap diff --git a/internal/k8s/builder/testdata/static-mapping.yaml b/internal/k8s/builder/testdata/static-mapping.yaml index 88e963a20..c2fe891c9 100644 --- a/internal/k8s/builder/testdata/static-mapping.yaml +++ b/internal/k8s/builder/testdata/static-mapping.yaml @@ -18,6 +18,11 @@ spec: envoy: "envoy:1" nodeSelector: "ingress-ready": "true" + tolerations: + - key: "key1" + operator: "Equal" + value: "value1" + effect: "NoSchedule" --- apiVersion: gateway.networking.k8s.io/v1alpha2 kind: GatewayClass diff --git a/pkg/apis/v1alpha1/types.go b/pkg/apis/v1alpha1/types.go index 753fc83af..dad2f70e7 100644 --- a/pkg/apis/v1alpha1/types.go +++ b/pkg/apis/v1alpha1/types.go @@ -39,6 +39,9 @@ type GatewayClassConfigSpec struct { // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ NodeSelector map[string]string `json:"nodeSelector,omitempty"` + //Toleration allow the scheduler to schedule nodes with matching tains + // More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // If this is set, then the Envoy container ports are mapped // to host ports. UseHostPorts bool `json:"useHostPorts,omitempty"` From f855fa057e606627ad5cf9d4f10f0c022e0cd157 Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Mon, 24 Oct 2022 14:26:59 -0500 Subject: [PATCH 02/13] generated files --- ...sul.hashicorp.com_gatewayclassconfigs.yaml | 41 +++++++++++++++++++ pkg/apis/v1alpha1/zz_generated.deepcopy.go | 7 ++++ 2 files changed, 48 insertions(+) diff --git a/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml b/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml index f01ae84e0..e9d6d54ce 100644 --- a/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml +++ b/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml @@ -149,6 +149,47 @@ spec: - NodePort - LoadBalancer type: string + tolerations: + description: 'Toleration allow the scheduler to schedule nodes with + matching tains More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/' + items: + description: The pod this Toleration is attached to tolerates any + taint that matches the triple using the matching + operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty + means match all taint effects. When specified, allowed values + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match all + values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the + value. Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod + can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time + the toleration (which must be of effect NoExecute, otherwise + this field is ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever (do not + evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array useHostPorts: description: If this is set, then the Envoy container ports are mapped to host ports. diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index d71ab8cde..4da5469c4 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -133,6 +133,13 @@ func (in *GatewayClassConfigSpec) DeepCopyInto(out *GatewayClassConfigSpec) { (*out)[key] = val } } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.ConsulSpec = in.ConsulSpec out.ImageSpec = in.ImageSpec in.CopyAnnotations.DeepCopyInto(&out.CopyAnnotations) From 44108d1edc62dc8286487ad47ceb2b5fd731c883 Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Tue, 25 Oct 2022 16:28:34 -0500 Subject: [PATCH 03/13] added matrix line --- .github/workflows/conformance.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 2f37b3efc..ba7026970 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -49,6 +49,11 @@ jobs: consul-image: "hashicorppreview/consul:1.14-dev" envoy-image: "envoyproxy/envoy:v1.22-latest" consul-k8s-version: "v0.49.0" + - name: "DELETE ME- TEST WITH BRANCH" + api-gateway-image: "hashicorppreview/consul-api-gateway:0.5-dev" + consul-image: "hashicorppreview/consul:1.14-dev" + envoy-image: "envoyproxy/envoy:v1.22-latest" + consul-k8s-version: "consul-api-gateway-add-tolerations-support" fail-fast: true name: "${{ matrix.cluster-type }} - ${{ matrix.config.name }}" concurrency: From dfebd1ccde5bc0718b5bb74b06e0857e478e687d Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Wed, 26 Oct 2022 10:25:51 -0500 Subject: [PATCH 04/13] add changelog entry --- .changelog/426.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/426.txt diff --git a/.changelog/426.txt b/.changelog/426.txt new file mode 100644 index 000000000..2da82693f --- /dev/null +++ b/.changelog/426.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +Add support for tolerations to Consul API Gateway Controller and GatewayClassConfig +``` From 02415da2f0dd2cc1612ec5d5678649ef1ddcc182 Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:08:04 -0500 Subject: [PATCH 05/13] Update config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml Co-authored-by: Nathan Coleman --- .../api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml b/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml index e9d6d54ce..782d4a440 100644 --- a/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml +++ b/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml @@ -151,7 +151,7 @@ spec: type: string tolerations: description: 'Toleration allow the scheduler to schedule nodes with - matching tains More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/' + matching taints More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/' items: description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching From 48859213e396b79db9d7bc40f735950e5d07b3da Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:08:11 -0500 Subject: [PATCH 06/13] Update pkg/apis/v1alpha1/types.go Co-authored-by: Nathan Coleman --- pkg/apis/v1alpha1/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/v1alpha1/types.go b/pkg/apis/v1alpha1/types.go index dad2f70e7..2e15e1d92 100644 --- a/pkg/apis/v1alpha1/types.go +++ b/pkg/apis/v1alpha1/types.go @@ -39,7 +39,7 @@ type GatewayClassConfigSpec struct { // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ NodeSelector map[string]string `json:"nodeSelector,omitempty"` - //Toleration allow the scheduler to schedule nodes with matching tains + // Tolerations allow the scheduler to schedule nodes with matching taints // More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // If this is set, then the Envoy container ports are mapped From b58782560e00ac2ed9e9906daa28fc0e9ceafdf5 Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Tue, 1 Nov 2022 10:42:38 -0500 Subject: [PATCH 07/13] regen files --- .../api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml b/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml index 782d4a440..c16bced68 100644 --- a/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml +++ b/config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml @@ -150,7 +150,7 @@ spec: - LoadBalancer type: string tolerations: - description: 'Toleration allow the scheduler to schedule nodes with + description: 'Tolerations allow the scheduler to schedule nodes with matching taints More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/' items: description: The pod this Toleration is attached to tolerates any From 42585e21d3acff0fd6d7ee63e612158592bda271 Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Tue, 1 Nov 2022 11:23:35 -0500 Subject: [PATCH 08/13] Update conformance.yml --- .github/workflows/conformance.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index ba7026970..2f37b3efc 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -49,11 +49,6 @@ jobs: consul-image: "hashicorppreview/consul:1.14-dev" envoy-image: "envoyproxy/envoy:v1.22-latest" consul-k8s-version: "v0.49.0" - - name: "DELETE ME- TEST WITH BRANCH" - api-gateway-image: "hashicorppreview/consul-api-gateway:0.5-dev" - consul-image: "hashicorppreview/consul:1.14-dev" - envoy-image: "envoyproxy/envoy:v1.22-latest" - consul-k8s-version: "consul-api-gateway-add-tolerations-support" fail-fast: true name: "${{ matrix.cluster-type }} - ${{ matrix.config.name }}" concurrency: From 0ddd9e23b0c313b97284176285d07af21b761e34 Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Thu, 3 Nov 2022 09:59:39 -0500 Subject: [PATCH 09/13] Update 426.txt --- .changelog/426.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/426.txt b/.changelog/426.txt index 2da82693f..6675518bb 100644 --- a/.changelog/426.txt +++ b/.changelog/426.txt @@ -1,3 +1,3 @@ ```release-note:enhancement -Add support for tolerations to Consul API Gateway Controller and GatewayClassConfig +Add support for tolerations to Consul API Gateway Controller and GatewayClassConfig. ``` From 591101e1384ab19ef93ceace2c72a19f2a1de300 Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Thu, 3 Nov 2022 12:08:34 -0500 Subject: [PATCH 10/13] prune test --- internal/testing/e2e/kind.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/internal/testing/e2e/kind.go b/internal/testing/e2e/kind.go index 4dc7390d4..1c96166f1 100644 --- a/internal/testing/e2e/kind.go +++ b/internal/testing/e2e/kind.go @@ -5,25 +5,24 @@ import ( "context" "errors" "fmt" + "github.com/hashicorp/consul/sdk/freeport" + "github.com/vladimirvivien/gexe" "html/template" "io" "io/ioutil" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "log" "os" "os/exec" - "strings" - "time" - - "github.com/hashicorp/consul/sdk/freeport" - "github.com/vladimirvivien/gexe" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/e2e-framework/klient" "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" + "strings" + "time" ) var ( @@ -219,6 +218,17 @@ func (k *kindCluster) Destroy() error { return fmt.Errorf("kind: remove config failed: %w", err) } + //todo move this into destroy if it works + timeoutContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(timeoutContext, "docker", "image", "prune", "-a", "-f") + var stdout, stderr bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + return err + } + return nil } @@ -291,6 +301,9 @@ func LoadKindDockerImage(clusterName string) env.Func { if err := loadImage(ctx, clusterName, DockerImage(ctx)); err != nil { return nil, err } + if err := loadImage(ctx, clusterName, "consul:local"); err != nil { + return nil, err + } for _, image := range ExtraDockerImages() { log.Printf("Loading additional docker image:%s into kind cluster", image) From 6cff241b24bfba6cfa4ca04f6163af7a06b63b0d Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Thu, 3 Nov 2022 12:22:38 -0500 Subject: [PATCH 11/13] Update kind.go --- internal/testing/e2e/kind.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/testing/e2e/kind.go b/internal/testing/e2e/kind.go index 1c96166f1..c940d9d29 100644 --- a/internal/testing/e2e/kind.go +++ b/internal/testing/e2e/kind.go @@ -301,10 +301,7 @@ func LoadKindDockerImage(clusterName string) env.Func { if err := loadImage(ctx, clusterName, DockerImage(ctx)); err != nil { return nil, err } - if err := loadImage(ctx, clusterName, "consul:local"); err != nil { - return nil, err - } - + for _, image := range ExtraDockerImages() { log.Printf("Loading additional docker image:%s into kind cluster", image) if err := loadImage(ctx, clusterName, image); err != nil { From d7526c806f3b176b82418efac81dafab5e1fe8e3 Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Thu, 3 Nov 2022 12:51:52 -0500 Subject: [PATCH 12/13] lint --- internal/testing/e2e/kind.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testing/e2e/kind.go b/internal/testing/e2e/kind.go index c940d9d29..178f17c91 100644 --- a/internal/testing/e2e/kind.go +++ b/internal/testing/e2e/kind.go @@ -301,7 +301,7 @@ func LoadKindDockerImage(clusterName string) env.Func { if err := loadImage(ctx, clusterName, DockerImage(ctx)); err != nil { return nil, err } - + for _, image := range ExtraDockerImages() { log.Printf("Loading additional docker image:%s into kind cluster", image) if err := loadImage(ctx, clusterName, image); err != nil { From 7c8c2108898ce336a5157d75ec49b88feabd4d27 Mon Sep 17 00:00:00 2001 From: Sarah Alsmiller Date: Thu, 3 Nov 2022 14:33:33 -0500 Subject: [PATCH 13/13] remove extra code --- internal/testing/e2e/kind.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/testing/e2e/kind.go b/internal/testing/e2e/kind.go index 178f17c91..09fc6e963 100644 --- a/internal/testing/e2e/kind.go +++ b/internal/testing/e2e/kind.go @@ -217,18 +217,6 @@ func (k *kindCluster) Destroy() error { if err := os.RemoveAll(k.config); err != nil { return fmt.Errorf("kind: remove config failed: %w", err) } - - //todo move this into destroy if it works - timeoutContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cmd := exec.CommandContext(timeoutContext, "docker", "image", "prune", "-a", "-f") - var stdout, stderr bytes.Buffer - cmd.Stderr = &stderr - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - return err - } - return nil }