From 5c3d7cfadfdc60f4597e769cdccadf18c1e00672 Mon Sep 17 00:00:00 2001 From: sujeet01 Date: Wed, 5 Feb 2025 03:25:27 +0530 Subject: [PATCH] Enhance `cert-manager` integration for metrics endpoints --- cmd/ironcore-controller-manager/main.go | 52 +++- config/apiserver/certmanager/certificate.yaml | 6 +- config/apiserver/default/kustomization.yaml | 6 +- config/apiserver/etcdless/kustomization.yaml | 6 +- ...tificate.yaml => certificate-metrics.yaml} | 10 +- .../certmanager/kustomization.yaml | 2 +- .../default/cert_metrics_manager_patch.yaml | 30 +++ .../default/kustomization.yaml | 243 +++++++++++------- .../default/manager_metrics_patch.yaml | 19 +- .../bucketpoollet-broker/manager/manager.yaml | 1 + .../prometheus/kustomization.yaml | 9 + .../prometheus/monitor.yaml | 7 +- .../prometheus/monitor_tls_patch.yaml | 22 ++ ...tificate.yaml => certificate-metrics.yaml} | 10 +- .../controller/certmanager/kustomization.yaml | 2 +- .../default/cert_metrics_manager_patch.yaml | 30 +++ config/controller/default/kustomization.yaml | 243 +++++++++++------- .../default/manager_metrics_patch.yaml | 20 +- config/controller/manager/manager.yaml | 3 + .../controller/prometheus/kustomization.yaml | 9 + config/controller/prometheus/monitor.yaml | 7 +- .../prometheus/monitor_tls_patch.yaml | 22 ++ ...tificate.yaml => certificate-metrics.yaml} | 10 +- .../certmanager/kustomization.yaml | 2 +- .../default/cert_metrics_manager_patch.yaml | 30 +++ .../default/kustomization.yaml | 66 ++++- .../default/manager_metrics_patch.yaml | 22 +- .../manager/manager.yaml | 1 + .../prometheus/kustomization.yaml | 9 + .../prometheus/monitor.yaml | 7 +- .../prometheus/monitor_tls_patch.yaml | 22 ++ ...tificate.yaml => certificate-metrics.yaml} | 10 +- .../certmanager/kustomization.yaml | 2 +- .../default/cert_metrics_manager_patch.yaml | 30 +++ .../default/kustomization.yaml | 243 +++++++++++------- .../default/manager_metrics_patch.yaml | 19 +- .../volumepoollet-broker/manager/manager.yaml | 1 + .../prometheus/kustomization.yaml | 9 + .../prometheus/monitor.yaml | 7 +- .../prometheus/monitor_tls_patch.yaml | 22 ++ .../cmd/bucketpoollet/app/app.go | 55 +++- .../cmd/machinepoollet/app/app.go | 56 +++- .../cmd/volumepoollet/app/app.go | 55 +++- 43 files changed, 1027 insertions(+), 410 deletions(-) rename config/bucketpoollet-broker/certmanager/{certificate.yaml => certificate-metrics.yaml} (65%) create mode 100644 config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml create mode 100644 config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml rename config/controller/certmanager/{certificate.yaml => certificate-metrics.yaml} (65%) create mode 100644 config/controller/default/cert_metrics_manager_patch.yaml create mode 100644 config/controller/prometheus/monitor_tls_patch.yaml rename config/machinepoollet-broker/certmanager/{certificate.yaml => certificate-metrics.yaml} (65%) create mode 100644 config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml create mode 100644 config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml rename config/volumepoollet-broker/certmanager/{certificate.yaml => certificate-metrics.yaml} (65%) create mode 100644 config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml create mode 100644 config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml diff --git a/cmd/ironcore-controller-manager/main.go b/cmd/ironcore-controller-manager/main.go index ee56f0044..9ade107e5 100644 --- a/cmd/ironcore-controller-manager/main.go +++ b/cmd/ironcore-controller-manager/main.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "time" corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" @@ -29,6 +30,7 @@ import ( quotaevaluatorironcore "github.com/ironcore-dev/ironcore/internal/quota/evaluator/ironcore" "github.com/ironcore-dev/ironcore/utils/quota" "k8s.io/utils/lru" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -103,6 +105,7 @@ func init() { func main() { var metricsAddr string var secureMetrics bool + var metricsCertPath, metricsCertName, metricsCertKey string var enableHTTP2 bool var enableLeaderElection bool var probeAddr string @@ -115,8 +118,12 @@ func main() { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics server") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -191,7 +198,7 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/controller/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -207,10 +214,37 @@ func main() { // can access the metrics endpoint. The RBAC are configured in 'config/controller/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/controller/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/controller/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -226,6 +260,14 @@ func main() { os.Exit(1) } + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + // Register controllers // compute controllers diff --git a/config/apiserver/certmanager/certificate.yaml b/config/apiserver/certmanager/certificate.yaml index 3e7cca677..9f96500b2 100644 --- a/config/apiserver/certmanager/certificate.yaml +++ b/config/apiserver/certmanager/certificate.yaml @@ -14,10 +14,10 @@ kind: Certificate metadata: name: apiserver-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system -spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize +spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local issuerRef: kind: Issuer name: apiserver-selfsigned-issuer diff --git a/config/apiserver/default/kustomization.yaml b/config/apiserver/default/kustomization.yaml index 6a930953d..d10a9fe8b 100644 --- a/config/apiserver/default/kustomization.yaml +++ b/config/apiserver/default/kustomization.yaml @@ -9,8 +9,10 @@ namespace: ironcore-system namePrefix: ironcore- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - ../rbac diff --git a/config/apiserver/etcdless/kustomization.yaml b/config/apiserver/etcdless/kustomization.yaml index 88cbecc3c..5d5129ef4 100644 --- a/config/apiserver/etcdless/kustomization.yaml +++ b/config/apiserver/etcdless/kustomization.yaml @@ -9,8 +9,10 @@ namespace: ironcore-system namePrefix: ironcore- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - ../rbac diff --git a/config/bucketpoollet-broker/certmanager/certificate.yaml b/config/bucketpoollet-broker/certmanager/certificate-metrics.yaml similarity index 65% rename from config/bucketpoollet-broker/certmanager/certificate.yaml rename to config/bucketpoollet-broker/certmanager/certificate-metrics.yaml index 1c64b4bb8..9fe7870c0 100644 --- a/config/bucketpoollet-broker/certmanager/certificate.yaml +++ b/config/bucketpoollet-broker/certmanager/certificate-metrics.yaml @@ -12,13 +12,13 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system -spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize +spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/bucketpoollet-broker/certmanager/kustomization.yaml b/config/bucketpoollet-broker/certmanager/kustomization.yaml index bebea5a59..622d85da4 100644 --- a/config/bucketpoollet-broker/certmanager/kustomization.yaml +++ b/config/bucketpoollet-broker/certmanager/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- certificate.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml b/config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..d97501553 --- /dev/null +++ b/config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/bucketpoollet-broker/default/kustomization.yaml b/config/bucketpoollet-broker/default/kustomization.yaml index 68b42b941..55e794a8c 100644 --- a/config/bucketpoollet-broker/default/kustomization.yaml +++ b/config/bucketpoollet-broker/default/kustomization.yaml @@ -9,105 +9,158 @@ namespace: bucketpoollet-system namePrefix: bucketpoollet- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - - ../broker-rbac - - ../manager -# - ../webhook -# - ../certmanager +- ../broker-rbac +- ../manager +# - ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +# - ../prometheus -patchesStrategicMerge: - - manager_metrics_patch.yaml +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment - # Mount the controller config file for loading manager configurations - # through a ComponentConfig type - #- manager_config_patch.yaml +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml - #- manager_webhook_patch.yaml - #- webhookcainjection_patch.yaml +#- manager_webhook_patch.yaml +#- webhookcainjection_patch.yaml -#replacements: -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +# replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true diff --git a/config/bucketpoollet-broker/default/manager_metrics_patch.yaml b/config/bucketpoollet-broker/default/manager_metrics_patch.yaml index 7f845a708..187e2d2b5 100644 --- a/config/bucketpoollet-broker/default/manager_metrics_patch.yaml +++ b/config/bucketpoollet-broker/default/manager_metrics_patch.yaml @@ -1,15 +1,4 @@ -# This patch modifies the controller manager deployment to enable the metrics endpoint over HTTPS. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=:8443" - - "--leader-elect" +# This patch adds an argument to the manager container to enable the metrics endpoint over HTTPS. +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/bucketpoollet-broker/manager/manager.yaml b/config/bucketpoollet-broker/manager/manager.yaml index d554771ae..846499b67 100644 --- a/config/bucketpoollet-broker/manager/manager.yaml +++ b/config/bucketpoollet-broker/manager/manager.yaml @@ -30,6 +30,7 @@ spec: - command: - /bucketpoollet args: + - --health-probe-bind-address=:8081 - --leader-elect image: bucketpoollet:latest name: manager diff --git a/config/bucketpoollet-broker/prometheus/kustomization.yaml b/config/bucketpoollet-broker/prometheus/kustomization.yaml index ed137168a..ebb5151bb 100644 --- a/config/bucketpoollet-broker/prometheus/kustomization.yaml +++ b/config/bucketpoollet-broker/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/bucketpoollet-broker/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +# patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/bucketpoollet-broker/prometheus/monitor.yaml b/config/bucketpoollet-broker/prometheus/monitor.yaml index d19136ae7..329e110d9 100644 --- a/config/bucketpoollet-broker/prometheus/monitor.yaml +++ b/config/bucketpoollet-broker/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml b/config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/controller/certmanager/certificate.yaml b/config/controller/certmanager/certificate-metrics.yaml similarity index 65% rename from config/controller/certmanager/certificate.yaml rename to config/controller/certmanager/certificate-metrics.yaml index 1c64b4bb8..9fe7870c0 100644 --- a/config/controller/certmanager/certificate.yaml +++ b/config/controller/certmanager/certificate-metrics.yaml @@ -12,13 +12,13 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system -spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize +spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/controller/certmanager/kustomization.yaml b/config/controller/certmanager/kustomization.yaml index bebea5a59..622d85da4 100644 --- a/config/controller/certmanager/kustomization.yaml +++ b/config/controller/certmanager/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- certificate.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/config/controller/default/cert_metrics_manager_patch.yaml b/config/controller/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..d97501553 --- /dev/null +++ b/config/controller/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/controller/default/kustomization.yaml b/config/controller/default/kustomization.yaml index 3401a9e16..bbe36f8ff 100644 --- a/config/controller/default/kustomization.yaml +++ b/config/controller/default/kustomization.yaml @@ -9,105 +9,158 @@ namespace: ironcore-system namePrefix: ironcore- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - - ../rbac - - ../manager - - ../webhook - - ../certmanager +- ../rbac +- ../manager +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +# - ../prometheus -patchesStrategicMerge: - - manager_metrics_patch.yaml +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment - # Mount the controller config file for loading manager configurations - # through a ComponentConfig type - #- manager_config_patch.yaml +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment - - manager_webhook_patch.yaml - # - webhookcainjection_patch.yaml +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml -replacements: - - source: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert - fieldPath: .metadata.namespace - targets: - - select: - kind: ValidatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 0 - create: true - - select: - kind: MutatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 0 - create: true - - source: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert - fieldPath: .metadata.name - targets: - - select: - kind: ValidatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 1 - create: true - - select: - kind: MutatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 1 - create: true - - source: - kind: Service - version: v1 - name: webhook-service - fieldPath: .metadata.name - targets: - - select: - kind: Certificate - group: cert-manager.io - version: v1 - fieldPaths: - - .spec.dnsNames.0 - - .spec.dnsNames.1 - options: - delimiter: '.' - index: 0 - create: true - - source: - kind: Service - version: v1 - name: webhook-service - fieldPath: .metadata.namespace - targets: - - select: - kind: Certificate - group: cert-manager.io - version: v1 - fieldPaths: - - .spec.dnsNames.0 - - .spec.dnsNames.1 - options: - delimiter: '.' - index: 1 - create: true +# - manager_webhook_patch.yaml +# - webhookcainjection_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +# replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true diff --git a/config/controller/default/manager_metrics_patch.yaml b/config/controller/default/manager_metrics_patch.yaml index 68970cc52..187e2d2b5 100644 --- a/config/controller/default/manager_metrics_patch.yaml +++ b/config/controller/default/manager_metrics_patch.yaml @@ -1,16 +1,4 @@ -# This patch modifies the controller manager deployment to enable the metrics endpoint over HTTPS. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=:8443" - - "--leader-elect" - - "--controllers=*" +# This patch adds an argument to the manager container to enable the metrics endpoint over HTTPS. +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/controller/manager/manager.yaml b/config/controller/manager/manager.yaml index a155b4cd7..d8ed3b9e7 100644 --- a/config/controller/manager/manager.yaml +++ b/config/controller/manager/manager.yaml @@ -30,6 +30,7 @@ spec: - command: - /ironcore-controller-manager args: + - --health-probe-bind-address=:8081 - --leader-elect - --controllers=* image: controller:latest @@ -55,5 +56,7 @@ spec: requests: cpu: 100m memory: 20Mi + volumeMounts: [] + volumes: [] serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/controller/prometheus/kustomization.yaml b/config/controller/prometheus/kustomization.yaml index ed137168a..0a8d2d6d3 100644 --- a/config/controller/prometheus/kustomization.yaml +++ b/config/controller/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/controller/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +# patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/controller/prometheus/monitor.yaml b/config/controller/prometheus/monitor.yaml index d19136ae7..431f3e7c3 100644 --- a/config/controller/prometheus/monitor.yaml +++ b/config/controller/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/controller/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/controller/prometheus/monitor_tls_patch.yaml b/config/controller/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/controller/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/machinepoollet-broker/certmanager/certificate.yaml b/config/machinepoollet-broker/certmanager/certificate-metrics.yaml similarity index 65% rename from config/machinepoollet-broker/certmanager/certificate.yaml rename to config/machinepoollet-broker/certmanager/certificate-metrics.yaml index 1c64b4bb8..9fe7870c0 100644 --- a/config/machinepoollet-broker/certmanager/certificate.yaml +++ b/config/machinepoollet-broker/certmanager/certificate-metrics.yaml @@ -12,13 +12,13 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system -spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize +spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/machinepoollet-broker/certmanager/kustomization.yaml b/config/machinepoollet-broker/certmanager/kustomization.yaml index bebea5a59..622d85da4 100644 --- a/config/machinepoollet-broker/certmanager/kustomization.yaml +++ b/config/machinepoollet-broker/certmanager/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- certificate.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml b/config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..d97501553 --- /dev/null +++ b/config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/machinepoollet-broker/default/kustomization.yaml b/config/machinepoollet-broker/default/kustomization.yaml index 6d6ff2454..d2825ed06 100644 --- a/config/machinepoollet-broker/default/kustomization.yaml +++ b/config/machinepoollet-broker/default/kustomization.yaml @@ -9,19 +9,34 @@ namespace: machinepoollet-system namePrefix: machinepoollet- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - ../broker-rbac - ../manager -# - ../webhook -# - ../certmanager +# - ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -patchesStrategicMerge: -- manager_metrics_patch.yaml +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment + # Mount the controller config file for loading manager configurations # through a ComponentConfig type @@ -30,7 +45,46 @@ patchesStrategicMerge: #- manager_webhook_patch.yaml #- webhookcainjection_patch.yaml +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true + - source: kind: Service group: "" diff --git a/config/machinepoollet-broker/default/manager_metrics_patch.yaml b/config/machinepoollet-broker/default/manager_metrics_patch.yaml index 3b6a5e215..187e2d2b5 100644 --- a/config/machinepoollet-broker/default/manager_metrics_patch.yaml +++ b/config/machinepoollet-broker/default/manager_metrics_patch.yaml @@ -1,18 +1,4 @@ -# This patch modifies the controller manager deployment to enable the metrics endpoint over HTTPS. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=:8443" - - "--leader-elect" - - "--machine-downward-api-label=root-machine-namespace=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-namespace']" - - "--machine-downward-api-label=root-machine-name=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-name']" - - "--machine-downward-api-label=root-machine-uid=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-uid']" +# This patch adds an argument to the manager container to enable the metrics endpoint over HTTPS. +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/machinepoollet-broker/manager/manager.yaml b/config/machinepoollet-broker/manager/manager.yaml index 528aee706..40f940e30 100644 --- a/config/machinepoollet-broker/manager/manager.yaml +++ b/config/machinepoollet-broker/manager/manager.yaml @@ -30,6 +30,7 @@ spec: - command: - /machinepoollet args: + - --health-probe-bind-address=:8081 - --leader-elect - --machine-downward-api-label=root-machine-namespace=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-namespace'] - --machine-downward-api-label=root-machine-name=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-name'] diff --git a/config/machinepoollet-broker/prometheus/kustomization.yaml b/config/machinepoollet-broker/prometheus/kustomization.yaml index ed137168a..2d71f5211 100644 --- a/config/machinepoollet-broker/prometheus/kustomization.yaml +++ b/config/machinepoollet-broker/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/machinepoollet-broker/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +# patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/machinepoollet-broker/prometheus/monitor.yaml b/config/machinepoollet-broker/prometheus/monitor.yaml index d19136ae7..c663b7335 100644 --- a/config/machinepoollet-broker/prometheus/monitor.yaml +++ b/config/machinepoollet-broker/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml b/config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/volumepoollet-broker/certmanager/certificate.yaml b/config/volumepoollet-broker/certmanager/certificate-metrics.yaml similarity index 65% rename from config/volumepoollet-broker/certmanager/certificate.yaml rename to config/volumepoollet-broker/certmanager/certificate-metrics.yaml index 1c64b4bb8..9fe7870c0 100644 --- a/config/volumepoollet-broker/certmanager/certificate.yaml +++ b/config/volumepoollet-broker/certmanager/certificate-metrics.yaml @@ -12,13 +12,13 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system -spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize +spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/volumepoollet-broker/certmanager/kustomization.yaml b/config/volumepoollet-broker/certmanager/kustomization.yaml index bebea5a59..622d85da4 100644 --- a/config/volumepoollet-broker/certmanager/kustomization.yaml +++ b/config/volumepoollet-broker/certmanager/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- certificate.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml b/config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..d97501553 --- /dev/null +++ b/config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/volumepoollet-broker/default/kustomization.yaml b/config/volumepoollet-broker/default/kustomization.yaml index 09d5e6b20..c53eddaae 100644 --- a/config/volumepoollet-broker/default/kustomization.yaml +++ b/config/volumepoollet-broker/default/kustomization.yaml @@ -9,105 +9,158 @@ namespace: volumepoollet-system namePrefix: volumepoollet- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - - ../broker-rbac - - ../manager -# - ../webhook -# - ../certmanager +- ../broker-rbac +- ../manager +# - ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +# - ../prometheus -patchesStrategicMerge: - - manager_metrics_patch.yaml +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment - # Mount the controller config file for loading manager configurations - # through a ComponentConfig type - #- manager_config_patch.yaml +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +# - manager_config_patch.yaml - #- manager_webhook_patch.yaml - #- webhookcainjection_patch.yaml +# - manager_webhook_patch.yaml +# - webhookcainjection_patch.yaml -#replacements: -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +# replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-cert +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true diff --git a/config/volumepoollet-broker/default/manager_metrics_patch.yaml b/config/volumepoollet-broker/default/manager_metrics_patch.yaml index 7f845a708..187e2d2b5 100644 --- a/config/volumepoollet-broker/default/manager_metrics_patch.yaml +++ b/config/volumepoollet-broker/default/manager_metrics_patch.yaml @@ -1,15 +1,4 @@ -# This patch modifies the controller manager deployment to enable the metrics endpoint over HTTPS. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=:8443" - - "--leader-elect" +# This patch adds an argument to the manager container to enable the metrics endpoint over HTTPS. +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/volumepoollet-broker/manager/manager.yaml b/config/volumepoollet-broker/manager/manager.yaml index 8d0f1793b..932f973a5 100644 --- a/config/volumepoollet-broker/manager/manager.yaml +++ b/config/volumepoollet-broker/manager/manager.yaml @@ -30,6 +30,7 @@ spec: - command: - /volumepoollet args: + - --health-probe-bind-address=:8081 - --leader-elect image: volumepoollet:latest name: manager diff --git a/config/volumepoollet-broker/prometheus/kustomization.yaml b/config/volumepoollet-broker/prometheus/kustomization.yaml index ed137168a..c68a1123a 100644 --- a/config/volumepoollet-broker/prometheus/kustomization.yaml +++ b/config/volumepoollet-broker/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/volumepoollet-broker/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +# patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/volumepoollet-broker/prometheus/monitor.yaml b/config/volumepoollet-broker/prometheus/monitor.yaml index d19136ae7..8f9bf8fe9 100644 --- a/config/volumepoollet-broker/prometheus/monitor.yaml +++ b/config/volumepoollet-broker/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml b/config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go b/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go index 843e37d36..c5f9d93fc 100644 --- a/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go +++ b/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go @@ -9,6 +9,7 @@ import ( goflag "flag" "fmt" "os" + "path/filepath" "time" "github.com/spf13/cobra" @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -51,6 +53,9 @@ type Options struct { GetConfigOptions config.GetConfigOptions MetricsAddr string SecureMetrics bool + MetricsCertPath string + MetricsCertName string + MetricsCertKey string EnableHTTP2 bool EnableLeaderElection bool LeaderElectionNamespace string @@ -79,8 +84,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") fs.BoolVar(&o.SecureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + fs.StringVar(&o.MetricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + fs.StringVar(&o.MetricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + fs.StringVar(&o.MetricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") fs.BoolVar(&o.EnableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics server") fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -187,7 +196,7 @@ func Run(ctx context.Context, opts Options) error { if !opts.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/bucketpoollet-broker/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -202,9 +211,37 @@ func Run(ctx context.Context, opts Options) error { // can access the metrics endpoint. The RBAC are configured in 'config/bucketpoollet-broker/broker-rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/bucketpoollet-broker/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/bucketpoollet-broker/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(opts.MetricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", opts.MetricsCertPath, "metrics-cert-name", opts.MetricsCertName, "metrics-cert-key", opts.MetricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(opts.MetricsCertPath, opts.MetricsCertName), + filepath.Join(opts.MetricsCertPath, opts.MetricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ @@ -224,6 +261,14 @@ func Run(ctx context.Context, opts Options) error { return err } + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + bucketClassMapper := bcm.NewGeneric(bucketRuntime, bcm.GenericOptions{}) if err := mgr.Add(bucketClassMapper); err != nil { return fmt.Errorf("error adding bucket class mapper: %w", err) diff --git a/poollet/machinepoollet/cmd/machinepoollet/app/app.go b/poollet/machinepoollet/cmd/machinepoollet/app/app.go index 7c30ae5b6..f160a09dd 100644 --- a/poollet/machinepoollet/cmd/machinepoollet/app/app.go +++ b/poollet/machinepoollet/cmd/machinepoollet/app/app.go @@ -9,6 +9,8 @@ import ( goflag "flag" "fmt" "net" + "os" + "path/filepath" "strconv" "time" @@ -40,6 +42,7 @@ import ( "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -61,6 +64,9 @@ type Options struct { GetConfigOptions config.GetConfigOptions MetricsAddr string SecureMetrics bool + MetricsCertPath string + MetricsCertName string + MetricsCertKey string EnableHTTP2 bool EnableLeaderElection bool LeaderElectionNamespace string @@ -95,8 +101,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") fs.BoolVar(&o.SecureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + fs.StringVar(&o.MetricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + fs.StringVar(&o.MetricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + fs.StringVar(&o.MetricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") fs.BoolVar(&o.EnableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics server") fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -243,7 +253,7 @@ func Run(ctx context.Context, opts Options) error { if !opts.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/machinepoollet-broker/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -258,9 +268,37 @@ func Run(ctx context.Context, opts Options) error { // can access the metrics endpoint. The RBAC are configured in 'config/machinepoollet-broker/broker-rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/machinepoollet-broker/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/machinepoollet-broker/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(opts.MetricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", opts.MetricsCertPath, "metrics-cert-name", opts.MetricsCertName, "metrics-cert-key", opts.MetricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(opts.MetricsCertPath, opts.MetricsCertName), + filepath.Join(opts.MetricsCertPath, opts.MetricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ @@ -290,6 +328,14 @@ func Run(ctx context.Context, opts Options) error { return err } + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + version, err := machineRuntime.Version(ctx, &iri.VersionRequest{}) if err != nil { return fmt.Errorf("error getting machine runtime version: %w", err) diff --git a/poollet/volumepoollet/cmd/volumepoollet/app/app.go b/poollet/volumepoollet/cmd/volumepoollet/app/app.go index eaef2f05c..1a4ae8919 100644 --- a/poollet/volumepoollet/cmd/volumepoollet/app/app.go +++ b/poollet/volumepoollet/cmd/volumepoollet/app/app.go @@ -9,6 +9,7 @@ import ( goflag "flag" "fmt" "os" + "path/filepath" "time" "github.com/spf13/cobra" @@ -32,6 +33,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -52,6 +54,9 @@ type Options struct { GetConfigOptions config.GetConfigOptions MetricsAddr string SecureMetrics bool + MetricsCertPath string + MetricsCertName string + MetricsCertKey string EnableHTTP2 bool EnableLeaderElection bool LeaderElectionNamespace string @@ -80,8 +85,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") fs.BoolVar(&o.SecureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + fs.StringVar(&o.MetricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + fs.StringVar(&o.MetricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + fs.StringVar(&o.MetricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") fs.BoolVar(&o.EnableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics server") fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -187,7 +196,7 @@ func Run(ctx context.Context, opts Options) error { if !opts.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/volumepoollet-broker/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -202,9 +211,37 @@ func Run(ctx context.Context, opts Options) error { // can access the metrics endpoint. The RBAC are configured in 'config/volumepoollet-broker/broker-rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/volumepoollet-broker/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/volumepoollet-broker/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(opts.MetricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", opts.MetricsCertPath, "metrics-cert-name", opts.MetricsCertName, "metrics-cert-key", opts.MetricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(opts.MetricsCertPath, opts.MetricsCertName), + filepath.Join(opts.MetricsCertPath, opts.MetricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ @@ -224,6 +261,14 @@ func Run(ctx context.Context, opts Options) error { return err } + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + volumeClassMapper := vcm.NewGeneric(volumeRuntime, vcm.GenericOptions{}) if err := mgr.Add(volumeClassMapper); err != nil { return fmt.Errorf("error adding volume class mapper: %w", err)