diff --git a/cmd/gardener-operator/app/app.go b/cmd/gardener-operator/app/app.go index 649dbf9e8ab..1639f735565 100644 --- a/cmd/gardener-operator/app/app.go +++ b/cmd/gardener-operator/app/app.go @@ -43,6 +43,7 @@ import ( "github.com/gardener/gardener/pkg/operator/apis/config" operatorclient "github.com/gardener/gardener/pkg/operator/client" "github.com/gardener/gardener/pkg/operator/controller" + "github.com/gardener/gardener/pkg/operator/metrics" "github.com/gardener/gardener/pkg/operator/webhook" ) @@ -186,6 +187,11 @@ func run(ctx context.Context, log logr.Logger, cfg *config.OperatorConfiguration return fmt.Errorf("failed adding webhook handlers to manager: %w", err) } + log.Info("Adding custom metrics to manager") + if err := metrics.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed adding metrics to manager: %w", err) + } + gardenClientMap, err := clientmapbuilder. NewGardenClientMapBuilder(). WithRuntimeClient(mgr.GetClient()). diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 88e9baa0f5d..de2c1e023f7 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -208,6 +208,7 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature. | DefaultSeccompProfile | `gardenlet`, `gardener-operator` | Enables the defaulting of the seccomp profile for Gardener managed workload in the garden or seed to `RuntimeDefault`. | | IPv6SingleStack | `gardener-apiserver`, `gardenlet` | Allows creating seed and shoot clusters with [IPv6 single-stack networking](../usage/ipv6.md) enabled in their spec ([GEP-21](../proposals/21-ipv6-singlestack-local.md)). If enabled in gardenlet, the default behavior is unchanged, but setting `ipFamilies=[IPv6]` in the `seedConfig` is allowed. Only if the `ipFamilies` setting is changed, gardenlet behaves differently. | | ShootForceDeletion | `gardener-apiserver` | Allows forceful deletion of Shoots by annotating them with the `confirmation.gardener.cloud/force-deletion` annotation. | +| DisableAPIServerProxyPort | `gardenlet` | Disables the proxy port (8443) on the istio-ingressgateway Services. It was previously used by the apiserver-proxy to route client traffic on the kubernetes Service to the corresponding API server using the TCP proxy protocol. As soon as a shoot has been reconciled by gardener v1.96+, the apiserver-proxy is reconfigured to use HTTP CONNECT on the tls-tunnel port (8132) instead, i.e., it reuses the reversed VPN path to connect to the correct API server. Operators can choose to remove the legacy apiserver-proxy port as soon as all shoots have switched to the new apiserver-proxy configuration. They might want to do so if they activate the ACL extension, which is vulnerable to proxy protocol headers of untrusted clients on the apiserver-proxy port. | | UseNamespacedCloudProfile | `gardener-apiserver` | Enables usage of `NamespacedCloudProfile`s in `Shoot`s. | | ShootManagedIssuer | `gardenlet` | Enables the shoot managed issuer functionality described in GEP 24. | | VPAForETCD | `gardenlet`, `gardener-operator` | Enables VPA for `etcd-main` and `etcd-events`, regardless of HVPA enablement. | diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 2f1bb45918a..daf0138df81 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -749,8 +749,9 @@ images: # External Authorization Server for the Istio Endpoint of Reversed VPN - name: ext-authz-server sourceRepository: github.com/gardener/ext-authz-server - repository: europe-docker.pkg.dev/gardener-project/releases/gardener/ext-authz-server - tag: "0.10.0" + # built from https://github.com/stackitcloud/ext-authz-server/tree/hackathon-apiserver-proxy + repository: ghcr.io/stackitcloud/ext-authz-server + tag: "0.11.0-dev-818656a" # API Server SNI - name: apiserver-proxy diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go b/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go index e74e504f0d6..164aa2311fa 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go @@ -129,7 +129,7 @@ ctr images mount "` + image + `" "$tmp_dir" echo "> Copy gardener-node-agent binary to host (/opt/bin) and make it executable" mkdir -p "/opt/bin" -cp -f "$tmp_dir/gardener-node-agent" "/opt/bin" +cp -f "$tmp_dir/ko-app/gardener-node-agent" "/opt/bin" chmod +x "/opt/bin/gardener-node-agent" echo "> Bootstrap gardener-node-agent" diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh b/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh index 38435aefe71..cbd4f872dd6 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh @@ -17,7 +17,7 @@ ctr images mount "{{ .image }}" "$tmp_dir" echo "> Copy gardener-node-agent binary to host ({{ .binaryDirectory }}) and make it executable" mkdir -p "{{ .binaryDirectory }}" -cp -f "$tmp_dir/gardener-node-agent" "{{ .binaryDirectory }}" +cp -f "$tmp_dir/ko-app/gardener-node-agent" "{{ .binaryDirectory }}" chmod +x "{{ .binaryDirectory }}/gardener-node-agent" echo "> Bootstrap gardener-node-agent" diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go index 6db6ed11d9c..95fc0af9c9b 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go @@ -79,7 +79,7 @@ func (component) Config(ctx components.Context) ([]extensionsv1alpha1.Unit, []ex Content: extensionsv1alpha1.FileContent{ ImageRef: &extensionsv1alpha1.FileContentImageRef{ Image: ctx.Images[imagevector.ContainerImageNameGardenerNodeAgent].String(), - FilePathInImage: "/gardener-node-agent", + FilePathInImage: "/ko-app/gardener-node-agent", }, }, }) diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go index 950c8dec199..ad4c1660a34 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go @@ -60,7 +60,7 @@ var _ = Describe("Component", func() { Content: extensionsv1alpha1.FileContent{ ImageRef: &extensionsv1alpha1.FileContentImageRef{ Image: "gardener-node-agent:v1", - FilePathInImage: "/gardener-node-agent", + FilePathInImage: "/ko-app/gardener-node-agent", }, }, }) diff --git a/pkg/component/gardener/apiserver/apiserver_test.go b/pkg/component/gardener/apiserver/apiserver_test.go index 78876e234e6..1d7add0c44d 100644 --- a/pkg/component/gardener/apiserver/apiserver_test.go +++ b/pkg/component/gardener/apiserver/apiserver_test.go @@ -584,6 +584,7 @@ var _ = Describe("GardenerAPIServer", func() { "--log-format=" + logFormat, "--secure-port=8443", "--workload-identity-token-issuer=" + workloadIdentityIssuer, + "--shoot-admin-kubeconfig-max-expiration=4320h", "--workload-identity-signing-key-file=/etc/gardener-apiserver/workload-identity/signing/key.pem", "--http2-max-streams-per-connection=1000", "--etcd-cafile=/srv/kubernetes/etcd/ca/bundle.crt", diff --git a/pkg/component/gardener/apiserver/deployment.go b/pkg/component/gardener/apiserver/deployment.go index 59c053c9d0d..3781e2213e8 100644 --- a/pkg/component/gardener/apiserver/deployment.go +++ b/pkg/component/gardener/apiserver/deployment.go @@ -101,6 +101,8 @@ func (g *gardenerAPIServer) deployment( "--log-format=" + g.values.LogFormat, fmt.Sprintf("--secure-port=%d", port), "--workload-identity-token-issuer=" + g.values.WorkloadIdentityTokenIssuer, + // TODO: replace this hardcoded configuration with proper fields in the Garden API + "--shoot-admin-kubeconfig-max-expiration=4320h", // 6 months }, Ports: []corev1.ContainerPort{{ Name: "https", diff --git a/pkg/component/gardener/controllermanager/configmaps.go b/pkg/component/gardener/controllermanager/configmaps.go index 07150368543..93d156c6747 100644 --- a/pkg/component/gardener/controllermanager/configmaps.go +++ b/pkg/component/gardener/controllermanager/configmaps.go @@ -62,6 +62,8 @@ func (g *gardenerControllerManager) configMapControllerManagerConfig() (*corev1. Project: &controllermanagerv1alpha1.ProjectControllerConfiguration{ ConcurrentSyncs: ptr.To(20), Quotas: g.values.Quotas, + // TODO: replace this hardcoded configuration with proper fields in the Garden API + StaleExpirationTimeDays: ptr.To(6000), }, SecretBinding: &controllermanagerv1alpha1.SecretBindingControllerConfiguration{ ConcurrentSyncs: ptr.To(20), diff --git a/pkg/component/gardener/controllermanager/controller_manager_test.go b/pkg/component/gardener/controllermanager/controller_manager_test.go index aa2c9238636..bdaeeb48fc9 100644 --- a/pkg/component/gardener/controllermanager/controller_manager_test.go +++ b/pkg/component/gardener/controllermanager/controller_manager_test.go @@ -328,7 +328,7 @@ var _ = Describe("GardenerControllerManager", func() { managedResourceSecretRuntime.Name = managedResourceRuntime.Spec.SecretRefs[0].Name Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretRuntime), managedResourceSecretRuntime)).To(Succeed()) cm := configMap(namespace, values) - Expect(cm.Name).To(Equal("gardener-controller-manager-config-960e3f19")) + Expect(cm.Name).To(Equal("gardener-controller-manager-config-625036ea")) expectedRuntimeObjects = []client.Object{ cm, serviceRuntime, @@ -720,8 +720,9 @@ func configMap(namespace string, testValues Values) *corev1.ConfigMap { ConcurrentSyncs: ptr.To(20), }, Project: &controllermanagerv1alpha1.ProjectControllerConfiguration{ - ConcurrentSyncs: ptr.To(20), - Quotas: testValues.Quotas, + ConcurrentSyncs: ptr.To(20), + Quotas: testValues.Quotas, + StaleExpirationTimeDays: ptr.To(6000), }, SecretBinding: &controllermanagerv1alpha1.SecretBindingControllerConfiguration{ ConcurrentSyncs: ptr.To(20), diff --git a/pkg/component/gardener/resourcemanager/resource_manager.go b/pkg/component/gardener/resourcemanager/resource_manager.go index f16500029f1..41c293c0448 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager.go +++ b/pkg/component/gardener/resourcemanager/resource_manager.go @@ -602,6 +602,9 @@ func (r *resourceManager) ensureConfigMap(ctx context.Context, configMap *corev1 }, r.values.NetworkPolicyAdditionalNamespaceSelectors...), IngressControllerSelector: r.values.NetworkPolicyControllerIngressControllerSelector, } + + config.SourceClientConnection.ClientConnectionConfiguration.QPS = 300 + config.SourceClientConnection.ClientConnectionConfiguration.Burst = 500 config.Webhooks.CRDDeletionProtection.Enabled = true config.Webhooks.ExtensionValidation.Enabled = true } diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index 5d567c933ba..43ba186540c 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -497,6 +497,8 @@ var _ = Describe("ResourceManager", func() { }, IngressControllerSelector: ingressControllerSelector, } + config.SourceClientConnection.ClientConnectionConfiguration.QPS = 300 + config.SourceClientConnection.ClientConnectionConfiguration.Burst = 500 config.Webhooks.CRDDeletionProtection.Enabled = true config.Webhooks.EndpointSliceHints.Enabled = true config.Webhooks.ExtensionValidation.Enabled = true diff --git a/pkg/component/kubernetes/apiserverexposure/sni.go b/pkg/component/kubernetes/apiserverexposure/sni.go index 8d92a9208ce..7dce8facab7 100644 --- a/pkg/component/kubernetes/apiserverexposure/sni.go +++ b/pkg/component/kubernetes/apiserverexposure/sni.go @@ -144,6 +144,10 @@ func (s *sni) Deploy(ctx context.Context) error { if err := managedresources.CreateForSeed(ctx, s.client, s.namespace, managedResourceName, false, serializedObjects); err != nil { return err } + } else { + if err := managedresources.DeleteForSeed(ctx, s.client, s.namespace, managedResourceName); err != nil { + return err + } } if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, s.client, destinationRule, istio.DestinationRuleWithLocalityPreference(destinationRule, getLabels(), hostName)); err != nil { diff --git a/pkg/component/kubernetes/apiserverexposure/sni_test.go b/pkg/component/kubernetes/apiserverexposure/sni_test.go index c6fa2b3cfb3..0cc8490cca6 100644 --- a/pkg/component/kubernetes/apiserverexposure/sni_test.go +++ b/pkg/component/kubernetes/apiserverexposure/sni_test.go @@ -219,7 +219,7 @@ var _ = Describe("#SNI", func() { if apiServerProxyValues != nil { managedResource := &resourcesv1alpha1.ManagedResource{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: expectedManagedResource.Namespace, Name: expectedManagedResource.Name}, managedResource)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKeyFromObject(expectedManagedResource), managedResource)).To(Succeed()) expectedManagedResource.Spec.SecretRefs = []corev1.LocalObjectReference{{Name: managedResource.Spec.SecretRefs[0].Name}} utilruntime.Must(references.InjectAnnotations(expectedManagedResource)) Expect(managedResource).To(DeepEqual(expectedManagedResource)) @@ -241,6 +241,8 @@ var _ = Describe("#SNI", func() { actualEnvoyFilter := managedResourceEnvoyFilter.(*istionetworkingv1alpha3.EnvoyFilter) // cannot validate the Spec as there is no meaningful way to unmarshal the data into the Golang structure Expect(actualEnvoyFilter.ObjectMeta).To(DeepEqual(expectedEnvoyFilterObjectMeta)) + } else { + Expect(c.Get(ctx, client.ObjectKeyFromObject(expectedManagedResource), &resourcesv1alpha1.ManagedResource{})).To(BeNotFoundError(), "should delete ManagedResource for apiserver-proxy EnvoyFilter") } } @@ -253,6 +255,10 @@ var _ = Describe("#SNI", func() { Context("when APIServer Proxy is not configured", func() { BeforeEach(func() { apiServerProxyValues = nil + + // create ManagedResource to ensure that Deploy deletes it + expectedManagedResource.ResourceVersion = "" + Expect(c.Create(ctx, expectedManagedResource)).To(Succeed()) }) It("should succeed deploying", func() { diff --git a/pkg/component/networking/apiserverproxy/apiserver_proxy.go b/pkg/component/networking/apiserverproxy/apiserver_proxy.go index b6cb5f69c73..a4cd068e8bf 100644 --- a/pkg/component/networking/apiserverproxy/apiserver_proxy.go +++ b/pkg/component/networking/apiserverproxy/apiserver_proxy.go @@ -27,6 +27,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" + "github.com/gardener/gardener/pkg/component/networking/vpn/seedserver" "github.com/gardener/gardener/pkg/component/observability/monitoring/prometheus/shoot" monitoringutils "github.com/gardener/gardener/pkg/component/observability/monitoring/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -43,7 +44,7 @@ const ( name = "apiserver-proxy" adminPort = 16910 - proxySeedServerPort = 8443 + proxySeedServerPort = seedserver.GatewayPort portNameMetrics = "metrics" volumeNameConfig = "proxy-config" @@ -207,6 +208,7 @@ func (a *apiserverProxy) computeResourcesData() (map[string][]byte, error) { "adminPort": adminPort, "proxySeedServerHost": a.values.ProxySeedServerHost, "proxySeedServerPort": proxySeedServerPort, + "namespace": a.namespace, }); err != nil { return nil, err } diff --git a/pkg/component/networking/apiserverproxy/apiserver_proxy_test.go b/pkg/component/networking/apiserverproxy/apiserver_proxy_test.go index 3b3edad9728..da8d01a9550 100644 --- a/pkg/component/networking/apiserverproxy/apiserver_proxy_test.go +++ b/pkg/component/networking/apiserverproxy/apiserver_proxy_test.go @@ -268,7 +268,7 @@ var _ = Describe("APIServerProxy", func() { Context("IPv4", func() { It("should deploy the managed resource successfully", func() { - test("607149fb") + test("e2e0f68e") }) }) @@ -279,7 +279,7 @@ var _ = Describe("APIServerProxy", func() { }) It("should deploy the managed resource successfully", func() { - test("3fdb1aaf") + test("5e1c6737") }) }) }) @@ -462,6 +462,12 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: kube_apiserver cluster: kube_apiserver + tunneling_config: + hostname: "api.internal.local.:443" + headers_to_add: + - header: + key: Reversed-VPN + value: "outbound|443||kube-apiserver.some-namespace.svc.cluster.local" access_log: - name: envoy.access_loggers.stdout typed_config: @@ -534,17 +540,7 @@ static_resources: address: socket_address: address: api.internal.local. - port_value: 8443 - transport_socket: - name: envoy.transport_sockets.upstream_proxy_protocol - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.proxy_protocol.v3.ProxyProtocolUpstreamTransport - config: - version: V2 - transport_socket: - name: envoy.transport_sockets.raw_buffer - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer + port_value: 8132 upstream_connection_options: tcp_keepalive: keepalive_time: 7200 diff --git a/pkg/component/networking/apiserverproxy/templates/envoy.yaml.tpl b/pkg/component/networking/apiserverproxy/templates/envoy.yaml.tpl index 9951b23d8c3..ab42aedab89 100644 --- a/pkg/component/networking/apiserverproxy/templates/envoy.yaml.tpl +++ b/pkg/component/networking/apiserverproxy/templates/envoy.yaml.tpl @@ -51,6 +51,12 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: kube_apiserver cluster: kube_apiserver + tunneling_config: + hostname: "{{ .proxySeedServerHost }}:443" + headers_to_add: + - header: + key: Reversed-VPN + value: "outbound|443||kube-apiserver.{{ .namespace }}.svc.cluster.local" access_log: - name: envoy.access_loggers.stdout typed_config: @@ -124,16 +130,6 @@ static_resources: socket_address: address: {{ .proxySeedServerHost }} port_value: {{ .proxySeedServerPort }} - transport_socket: - name: envoy.transport_sockets.upstream_proxy_protocol - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.proxy_protocol.v3.ProxyProtocolUpstreamTransport - config: - version: V2 - transport_socket: - name: envoy.transport_sockets.raw_buffer - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer upstream_connection_options: tcp_keepalive: keepalive_time: 7200 diff --git a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml index 1a4bed9a2a7..634f0e2396a 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml +++ b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml @@ -6,7 +6,7 @@ metrics_path: /metrics/cadvisor tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: {{.IsManagedSeed}} + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: diff --git a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml index 8a901530729..b984c74240c 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml +++ b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml @@ -4,7 +4,7 @@ scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: {{.IsManagedSeed}} + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: diff --git a/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go b/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go index 86f5fa66dcf..8c935992cd5 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go +++ b/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go @@ -122,7 +122,7 @@ metrics_path: /metrics/cadvisor tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: false + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: @@ -183,7 +183,7 @@ scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: false + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: diff --git a/pkg/component/shared/resourcemanager.go b/pkg/component/shared/resourcemanager.go index 3e5c26d94a4..32534497814 100644 --- a/pkg/component/shared/resourcemanager.go +++ b/pkg/component/shared/resourcemanager.go @@ -66,6 +66,7 @@ func NewRuntimeGardenerResourceManager( return resourcemanager.New(c, gardenNamespaceName, secretsManager, resourcemanager.Values{ ConcurrentSyncs: ptr.To(20), + AlwaysUpdate: ptr.To(true), DefaultSeccompProfileEnabled: defaultSeccompProfileEnabled, DefaultNotReadyToleration: defaultNotReadyToleration, DefaultUnreachableToleration: defaultUnreachableToleration, diff --git a/pkg/features/features.go b/pkg/features/features.go index c669397a77f..b19d87b5d36 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -62,6 +62,18 @@ const ( // alpha: v1.93.0 ShootManagedIssuer featuregate.Feature = "ShootManagedIssuer" + // DisableAPIServerProxyPort disables the proxy port (8443) on the istio-ingressgateway Services. It was previously + // used by the apiserver-proxy to route client traffic on the kubernetes Service to the corresponding API server using + // the TCP proxy protocol. + // As soon as a shoot has been reconciled by gardener v1.96+, the apiserver-proxy is reconfigured to use HTTP CONNECT + // on the tls-tunnel port (8132) instead, i.e., it reuses the reversed VPN path to connect to the correct API server. + // Operators can choose to remove the legacy apiserver-proxy port as soon as all shoots have switched to the new + // apiserver-proxy configuration. They might want to do so if they activate the ACL extension, which is vulnerable to + // proxy protocol headers of untrusted clients on the apiserver-proxy port. + // owner: @timebertt + // alpha: v1.96.0 + DisableAPIServerProxyPort = "DisableAPIServerProxyPort" + // VPAAndHPAForAPIServer an autoscaling mechanism for kube-apiserver of shoot or virtual garden clusters, and the gardener-apiserver. // They are scaled simultaneously by VPA and HPA on the same metric (CPU and memory usage). // The pod-trashing cycle between VPA and HPA scaling on the same metric is avoided @@ -120,6 +132,7 @@ var AllFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ ShootManagedIssuer: {Default: false, PreRelease: featuregate.Alpha}, ShootForceDeletion: {Default: true, PreRelease: featuregate.Beta}, UseNamespacedCloudProfile: {Default: false, PreRelease: featuregate.Alpha}, + DisableAPIServerProxyPort: {Default: false, PreRelease: featuregate.Alpha}, VPAAndHPAForAPIServer: {Default: true, PreRelease: featuregate.Beta}, ShootCredentialsBinding: {Default: false, PreRelease: featuregate.Alpha}, NewWorkerPoolHash: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index 6fbc8b06a98..c615342e4d5 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -272,6 +272,18 @@ func (r *Reconciler) newGardenerResourceManager(seed *gardencorev1beta1.Seed, se func (r *Reconciler) newIstio(ctx context.Context, seed *seedpkg.Seed, isGardenCluster bool) (component.DeployWaiter, map[string]string, string, error) { labels := sharedcomponent.GetIstioZoneLabels(r.Config.SNI.Ingress.Labels, nil) + ports := []corev1.ServicePort{ + {Name: "tcp", Port: 443, TargetPort: intstr.FromInt32(9443)}, + {Name: "tls-tunnel", Port: vpnseedserver.GatewayPort, TargetPort: intstr.FromInt32(vpnseedserver.GatewayPort)}, + } + + proxyProtocolEnabled := !features.DefaultFeatureGate.Enabled(features.DisableAPIServerProxyPort) + if proxyProtocolEnabled { + ports = append(ports, corev1.ServicePort{ + Name: "proxy", Port: 8443, TargetPort: intstr.FromInt32(8443), + }) + } + istioDeployer, err := sharedcomponent.NewIstio( ctx, r.SeedClientSet.Client(), @@ -285,12 +297,8 @@ func (r *Reconciler) newIstio(ctx context.Context, seed *seedpkg.Seed, isGardenC seed.GetLoadBalancerServiceAnnotations(), seed.GetLoadBalancerServiceExternalTrafficPolicy(), r.Config.SNI.Ingress.ServiceExternalIP, - []corev1.ServicePort{ - {Name: "proxy", Port: 8443, TargetPort: intstr.FromInt32(8443)}, - {Name: "tcp", Port: 443, TargetPort: intstr.FromInt32(9443)}, - {Name: "tls-tunnel", Port: vpnseedserver.GatewayPort, TargetPort: intstr.FromInt32(vpnseedserver.GatewayPort)}, - }, - true, + ports, + proxyProtocolEnabled, seed.GetLoadBalancerServiceProxyProtocolTermination(), true, seed.GetInfo().Spec.Provider.Zones, diff --git a/pkg/gardenlet/features/features.go b/pkg/gardenlet/features/features.go index 070f89854fd..2717abe29a1 100644 --- a/pkg/gardenlet/features/features.go +++ b/pkg/gardenlet/features/features.go @@ -25,6 +25,7 @@ func GetFeatures() []featuregate.Feature { features.DefaultSeccompProfile, features.IPv6SingleStack, features.ShootManagedIssuer, + features.DisableAPIServerProxyPort, features.VPAAndHPAForAPIServer, features.NewWorkerPoolHash, } diff --git a/pkg/gardenlet/operation/botanist/kubeapiserverexposure.go b/pkg/gardenlet/operation/botanist/kubeapiserverexposure.go index 83bf06f4ce7..cf1096bb841 100644 --- a/pkg/gardenlet/operation/botanist/kubeapiserverexposure.go +++ b/pkg/gardenlet/operation/botanist/kubeapiserverexposure.go @@ -13,6 +13,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/component" kubeapiserverexposure "github.com/gardener/gardener/pkg/component/kubernetes/apiserverexposure" + "github.com/gardener/gardener/pkg/features" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -87,7 +88,7 @@ func (b *Botanist) setAPIServerServiceClusterIP(clusterIP string) { v1beta1constants.DeploymentNameKubeAPIServer, b.Shoot.SeedNamespace, func() *kubeapiserverexposure.SNIValues { - return &kubeapiserverexposure.SNIValues{ + values := &kubeapiserverexposure.SNIValues{ Hosts: []string{ gardenerutils.GetAPIServerDomain(*b.Shoot.ExternalClusterDomain), gardenerutils.GetAPIServerDomain(b.Shoot.InternalClusterDomain), @@ -101,6 +102,12 @@ func (b *Botanist) setAPIServerServiceClusterIP(clusterIP string) { Labels: b.IstioLabels(), }, } + + if features.DefaultFeatureGate.Enabled(features.DisableAPIServerProxyPort) { + values.APIServerProxy = nil + } + + return values }, ) } diff --git a/pkg/operator/metrics/garden.go b/pkg/operator/metrics/garden.go new file mode 100644 index 00000000000..bd9c727edcc --- /dev/null +++ b/pkg/operator/metrics/garden.go @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" +) + +const gardenSubsystem = "garden" + +type gardenCollector struct { + runtimeClient client.Reader + log logr.Logger + + condition *prometheus.Desc + operationSucceeded *prometheus.Desc +} + +func newGardenCollector(k8sClient client.Reader, log logr.Logger) *gardenCollector { + c := &gardenCollector{ + runtimeClient: k8sClient, + log: log, + } + c.setMetricDefinitions() + return c +} + +func (c *gardenCollector) setMetricDefinitions() { + c.condition = prometheus.NewDesc( + prometheus.BuildFQName(metricPrefix, gardenSubsystem, "condition"), + "Condition state of the Garden.", + []string{ + "name", + "condition", + "status", + }, + nil, + ) + c.operationSucceeded = prometheus.NewDesc( + prometheus.BuildFQName(metricPrefix, gardenSubsystem, "operation_succeeded"), + "Returns 1 if the last operation state is Succeeded.", + []string{ + "name", + }, + nil, + ) +} + +func (c *gardenCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.condition + ch <- c.operationSucceeded +} + +func (c *gardenCollector) Collect(ch chan<- prometheus.Metric) { + ctx := context.Background() + + gardenList := &operatorv1alpha1.GardenList{} + if err := c.runtimeClient.List(ctx, gardenList); err != nil { + c.log.Error(err, "Failed to list gardens") + return + } + + for _, garden := range gardenList.Items { + c.collectConditionMetric(ch, &garden) + c.collectOperationMetric(ch, &garden) + } +} + +func (c gardenCollector) collectConditionMetric(ch chan<- prometheus.Metric, garden *operatorv1alpha1.Garden) { + for _, condition := range garden.Status.Conditions { + if condition.Type == "" { + continue + } + for _, status := range []gardencorev1beta1.ConditionStatus{ + gardencorev1beta1.ConditionFalse, + gardencorev1beta1.ConditionTrue, + gardencorev1beta1.ConditionProgressing, + gardencorev1beta1.ConditionUnknown, + } { + val := float64(0) + if condition.Status == status { + val = 1 + } + ch <- prometheus.MustNewConstMetric( + c.condition, + prometheus.GaugeValue, + val, + []string{ + garden.Name, + string(condition.Type), + string(status), + }..., + ) + } + } +} + +func (c *gardenCollector) collectOperationMetric(ch chan<- prometheus.Metric, garden *operatorv1alpha1.Garden) { + if garden.Status.LastOperation == nil { + return + } + val := float64(0) + if garden.Status.LastOperation.State == gardencorev1beta1.LastOperationStateSucceeded { + val = 1 + } + ch <- prometheus.MustNewConstMetric( + c.operationSucceeded, + prometheus.GaugeValue, + val, + []string{ + garden.Name, + }..., + ) +} diff --git a/pkg/operator/metrics/garden_test.go b/pkg/operator/metrics/garden_test.go new file mode 100644 index 00000000000..db591f14357 --- /dev/null +++ b/pkg/operator/metrics/garden_test.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "context" + "strings" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" +) + +var _ = Describe("Garden metrics", func() { + var ( + ctx context.Context + k8sClient client.Client + + c prometheus.Collector + garden *operatorv1alpha1.Garden + ) + + BeforeEach(func() { + testScheme := runtime.NewScheme() + Expect(operatorv1alpha1.AddToScheme(testScheme)).To(Succeed()) + k8sClient = fake.NewClientBuilder(). + WithScheme(testScheme). + WithStatusSubresource(&operatorv1alpha1.Garden{}). + Build() + + c = newGardenCollector(k8sClient, logr.Discard()) + + garden = &operatorv1alpha1.Garden{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + Expect(k8sClient.Create(ctx, garden)).To(Succeed()) + + garden.Status = operatorv1alpha1.GardenStatus{ + LastOperation: &gardencorev1beta1.LastOperation{ + State: gardencorev1beta1.LastOperationStateSucceeded, + }, + Conditions: []gardencorev1beta1.Condition{ + {Type: operatorv1alpha1.RuntimeComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, + {Type: operatorv1alpha1.VirtualComponentsHealthy, Status: gardencorev1beta1.ConditionFalse}, + }, + } + Expect(k8sClient.Status().Update(ctx, garden)).To(Succeed()) + + }) + + It("should collect condition metrics", func() { + expected := strings.NewReader(`# HELP gardener_operator_garden_condition Condition state of the Garden. +# TYPE gardener_operator_garden_condition gauge +gardener_operator_garden_condition{condition="RuntimeComponentsHealthy",name="foo",status="False"} 0 +gardener_operator_garden_condition{condition="RuntimeComponentsHealthy",name="foo",status="Progressing"} 0 +gardener_operator_garden_condition{condition="RuntimeComponentsHealthy",name="foo",status="True"} 1 +gardener_operator_garden_condition{condition="RuntimeComponentsHealthy",name="foo",status="Unknown"} 0 +gardener_operator_garden_condition{condition="VirtualComponentsHealthy",name="foo",status="False"} 1 +gardener_operator_garden_condition{condition="VirtualComponentsHealthy",name="foo",status="Progressing"} 0 +gardener_operator_garden_condition{condition="VirtualComponentsHealthy",name="foo",status="True"} 0 +gardener_operator_garden_condition{condition="VirtualComponentsHealthy",name="foo",status="Unknown"} 0 +`) + + Expect( + testutil.CollectAndCompare(c, expected, "gardener_operator_garden_condition"), + ).To(Succeed()) + }) + + It("should collect operation succeeded metrics", func() { + expected := strings.NewReader(`# HELP gardener_operator_garden_operation_succeeded Returns 1 if the last operation state is Succeeded. +# TYPE gardener_operator_garden_operation_succeeded gauge +gardener_operator_garden_operation_succeeded{name="foo"} 1 +`) + + Expect( + testutil.CollectAndCompare(c, expected, "gardener_operator_garden_operation_succeeded"), + ).To(Succeed()) + }) + + It("should collect the metric for not succeeded gardens", func() { + garden.Status.LastOperation.State = gardencorev1beta1.LastOperationStateError + Expect(k8sClient.Status().Update(ctx, garden)).To(Succeed()) + + expected := strings.NewReader(`# HELP gardener_operator_garden_operation_succeeded Returns 1 if the last operation state is Succeeded. +# TYPE gardener_operator_garden_operation_succeeded gauge +gardener_operator_garden_operation_succeeded{name="foo"} 0 +`) + + Expect( + testutil.CollectAndCompare(c, expected, "gardener_operator_garden_operation_succeeded"), + ).To(Succeed()) + }) +}) diff --git a/pkg/operator/metrics/metrics.go b/pkg/operator/metrics/metrics.go new file mode 100644 index 00000000000..196c25b224b --- /dev/null +++ b/pkg/operator/metrics/metrics.go @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/manager" + runtimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + metricPrefix = "gardener_operator" +) + +type runnable struct { + gardenCollector *gardenCollector +} + +// AddToManager adds the custom metrics collectors to the metrics registry. This is done "inside" a `manager.Runnable`, +// because that guarantees that the cache informers are synced, before the metrics are added / scraped for the first +// time. +func AddToManager(_ context.Context, mgr manager.Manager) error { + k8sClient := mgr.GetClient() + return mgr.Add(&runnable{ + gardenCollector: newGardenCollector(k8sClient, mgr.GetLogger().WithName("operator-metrics")), + }) +} + +func (r *runnable) Start(_ context.Context) error { + runtimemetrics.Registry.MustRegister(r.gardenCollector) + return nil +} diff --git a/pkg/operator/metrics/metrics_suite_test.go b/pkg/operator/metrics/metrics_suite_test.go new file mode 100644 index 00000000000..6d03963d547 --- /dev/null +++ b/pkg/operator/metrics/metrics_suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package metrics_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Operator Metrics Suite") +} diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index b54cf8012df..2920ab680a8 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -240,6 +240,7 @@ build: - pkg/operator/controller/gardenlet - pkg/operator/features - pkg/operator/predicate + - pkg/operator/metrics - pkg/operator/webhook - pkg/operator/webhook/defaulting - pkg/operator/webhook/validation