Skip to content

Commit

Permalink
Detect admission registration API version (#4569)
Browse files Browse the repository at this point in the history
Detect admission registration API version. admissionregistration/v1 is used when available. On older versions of K8S (mostly OCP 3.11) the operator attempts to use v1beta1.
  • Loading branch information
barkbay authored Jun 18, 2021
1 parent 1ea2775 commit db29820
Show file tree
Hide file tree
Showing 6 changed files with 220 additions and 48 deletions.
13 changes: 10 additions & 3 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -748,13 +748,20 @@ func setupWebhook(mgr manager.Manager, certRotation certificates.RotationParams,
Rotation: certRotation,
}

// retrieve the current webhook configuration interface
wh, err := webhookParams.NewAdmissionControllerInterface(context.Background(), clientset)
if err != nil {
log.Error(err, "unable to setup the webhook certificates")
os.Exit(1)
}

// Force a first reconciliation to create the resources before the server is started
if err := webhookParams.ReconcileResources(context.Background(), clientset); err != nil {
log.Error(err, "unable to setup and fill the webhook certificates")
if err := webhookParams.ReconcileResources(context.Background(), clientset, wh); err != nil {
log.Error(err, "unable to setup the webhook certificates")
os.Exit(1)
}

if err := webhook.Add(mgr, webhookParams, clientset); err != nil {
if err := webhook.Add(mgr, webhookParams, clientset, wh); err != nil {
log.Error(err, "unable to create controller", "controller", webhook.ControllerName)
os.Exit(1)
}
Expand Down
162 changes: 162 additions & 0 deletions pkg/controller/webhook/admission_registration.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package webhook

import (
"context"

v1 "k8s.io/api/admissionregistration/v1"
"k8s.io/api/admissionregistration/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type webhook struct {
webhookConfigurationName, webhookName string
caBundle []byte
}

type Services map[types.NamespacedName]struct{}

// AdmissionControllerInterface helps to setup webhooks for different versions of the admissionregistration API.
type AdmissionControllerInterface interface {
getType() client.Object
// services returns the set of services used by the Webhooks
services() Services
// webhooks returns the list of webhooks in the configuration
webhooks() []webhook
// updateCABundle updates CABundle with the provided CA in all the Webhooks
updateCABundle(caCert []byte) error
}

func (w *Params) NewAdmissionControllerInterface(ctx context.Context, clientset kubernetes.Interface) (AdmissionControllerInterface, error) {
// Detect if V1 is available
_, err := clientset.Discovery().ServerResourcesForGroupVersion(v1.SchemeGroupVersion.String())
if errors.IsNotFound(err) { // Presumably a K8S cluster older than 1.16
log.V(1).Info("admissionregistration.k8s.io/v1 is not available, using v1beta1 for webhook configuration")
webhookConfiguration, err := clientset.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
if err != nil {
// 404 is also considered as an error, webhook configuration is expected to be created before the operator is started
return nil, err
}
return &v1beta1webhookHandler{ctx: ctx, clientset: clientset, webhookConfiguration: webhookConfiguration}, nil
}
if err != nil {
return nil, err
}
log.V(1).Info("using admissionregistration.k8s.io/v1 for webhook configuration")
webhookConfiguration, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
if err != nil {
// 404 is also considered as an error, webhook configuration is expected to be created before the operator is started
return nil, err
}
return &v1webhookHandler{ctx: ctx, clientset: clientset, webhookConfiguration: webhookConfiguration}, nil
}

// - admissionregistration.k8s.io/v1 implementation

var _ AdmissionControllerInterface = &v1webhookHandler{}

type v1webhookHandler struct {
clientset kubernetes.Interface
ctx context.Context
webhookConfiguration *v1.ValidatingWebhookConfiguration
}

func (*v1webhookHandler) getType() client.Object {
return &v1.ValidatingWebhookConfiguration{}
}

func (v1w *v1webhookHandler) webhooks() []webhook {
webhooks := make([]webhook, 0, len(v1w.webhookConfiguration.Webhooks))
for _, wh := range v1w.webhookConfiguration.Webhooks {
webhook := webhook{
webhookConfigurationName: v1w.webhookConfiguration.Name,
webhookName: wh.Name,
caBundle: wh.ClientConfig.CABundle,
}
webhooks = append(webhooks, webhook)
}
return webhooks
}

func (v1w *v1webhookHandler) services() Services {
services := make(map[types.NamespacedName]struct{})
for _, wh := range v1w.webhookConfiguration.Webhooks {
if wh.ClientConfig.Service == nil {
continue
}
services[types.NamespacedName{
Namespace: wh.ClientConfig.Service.Namespace,
Name: wh.ClientConfig.Service.Name,
}] = struct{}{}
}
return services
}

func (v1w *v1webhookHandler) updateCABundle(caCert []byte) error {
for i := range v1w.webhookConfiguration.Webhooks {
v1w.webhookConfiguration.Webhooks[i].ClientConfig.CABundle = caCert
}
_, err := v1w.clientset.
AdmissionregistrationV1().
ValidatingWebhookConfigurations().
Update(v1w.ctx, v1w.webhookConfiguration, metav1.UpdateOptions{})
return err
}

// - admissionregistration.k8s.io/v1beta1 implementation

var _ AdmissionControllerInterface = &v1beta1webhookHandler{}

type v1beta1webhookHandler struct {
clientset kubernetes.Interface
ctx context.Context
webhookConfiguration *v1beta1.ValidatingWebhookConfiguration
}

func (*v1beta1webhookHandler) getType() client.Object {
return &v1beta1.ValidatingWebhookConfiguration{}
}

func (v1beta1w *v1beta1webhookHandler) webhooks() []webhook {
webhooks := make([]webhook, 0, len(v1beta1w.webhookConfiguration.Webhooks))
for _, wh := range v1beta1w.webhookConfiguration.Webhooks {
webhook := webhook{
webhookConfigurationName: v1beta1w.webhookConfiguration.Name,
caBundle: wh.ClientConfig.CABundle,
}
webhooks = append(webhooks, webhook)
}
return webhooks
}

func (v1beta1w *v1beta1webhookHandler) services() Services {
services := make(map[types.NamespacedName]struct{})
for _, wh := range v1beta1w.webhookConfiguration.Webhooks {
if wh.ClientConfig.Service == nil {
continue
}
services[types.NamespacedName{
Namespace: wh.ClientConfig.Service.Namespace,
Name: wh.ClientConfig.Service.Name,
}] = struct{}{}
}
return services
}

func (v1beta1w *v1beta1webhookHandler) updateCABundle(caCert []byte) error {
for i := range v1beta1w.webhookConfiguration.Webhooks {
v1beta1w.webhookConfiguration.Webhooks[i].ClientConfig.CABundle = caCert
}
_, err := v1beta1w.clientset.
AdmissionregistrationV1beta1().
ValidatingWebhookConfigurations().
Update(v1beta1w.ctx, v1beta1w.webhookConfiguration, metav1.UpdateOptions{})
return err
}
29 changes: 12 additions & 17 deletions pkg/controller/webhook/cert.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"crypto/x509/pkix"
"time"

"k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

Expand All @@ -27,7 +26,7 @@ type WebhookCertificates struct {
serverCert []byte
}

func (w *Params) shouldRenewCertificates(serverCertificates *corev1.Secret, webhookConfiguration *v1beta1.ValidatingWebhookConfiguration) bool {
func (w *Params) shouldRenewCertificates(serverCertificates *corev1.Secret, webhooks []webhook) bool {
// Read the current certificate used by the server
serverCA := certificates.BuildCAFromSecret(*serverCertificates)
if serverCA == nil {
Expand All @@ -37,15 +36,15 @@ func (w *Params) shouldRenewCertificates(serverCertificates *corev1.Secret, webh
return true
}
// Read the certificate in the webhook configuration
for _, webhook := range webhookConfiguration.Webhooks {
caBytes := webhook.ClientConfig.CABundle
for _, webhook := range webhooks {
caBytes := webhook.caBundle
if len(caBytes) == 0 {
return true
}
// Parse the certificates
certs, err := certificates.ParsePEMCerts(caBytes)
if err != nil {
log.Error(err, "Cannot parse PEM cert from webhook configuration, will create a new one", "webhook_name", webhookConfiguration.Name)
log.Error(err, "Cannot parse PEM cert from webhook configuration, will create a new one", "webhook_name", webhook.webhookConfigurationName)
return true
}
if len(certs) == 0 {
Expand All @@ -65,7 +64,7 @@ func (w *Params) shouldRenewCertificates(serverCertificates *corev1.Secret, webh
// certificate of the webhook server.
// The private key is not retained or persisted, all the artifacts are regenerated and updated if needed when the
// certificate is about to expire or is missing.
func (w *Params) newCertificates(webhookConf *v1beta1.ValidatingWebhookConfiguration) (WebhookCertificates, error) {
func (w *Params) newCertificates(webhookServices Services) (WebhookCertificates, error) {
webhookCertificates := WebhookCertificates{}

// Create a new CA
Expand Down Expand Up @@ -102,7 +101,7 @@ func (w *Params) newCertificates(webhookConf *v1beta1.ValidatingWebhookConfigura
CommonName: "elastic-webhook",
OrganizationalUnit: []string{"elastic-webhook"},
},
DNSNames: extractDNSNames(webhookConf),
DNSNames: extractDNSNames(webhookServices),
NotBefore: time.Now().Add(-10 * time.Minute),
NotAfter: time.Now().Add(w.Rotation.Validity),
PublicKeyAlgorithm: parsedCSR.PublicKeyAlgorithm,
Expand All @@ -121,16 +120,12 @@ func (w *Params) newCertificates(webhookConf *v1beta1.ValidatingWebhookConfigura
return webhookCertificates, nil
}

func extractDNSNames(webhookConf *v1beta1.ValidatingWebhookConfiguration) []string {
svcNames := make(map[string]struct{}, len(webhookConf.Webhooks))

for _, wh := range webhookConf.Webhooks {
svcRef := wh.ClientConfig.Service
if svcRef == nil {
continue
}

names := k8s.GetServiceDNSName(corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: svcRef.Namespace, Name: svcRef.Name}})
func extractDNSNames(webhookServices Services) []string {
svcNames := make(map[string]struct{}, len(webhookServices))
for svcRef := range webhookServices {
names := k8s.GetServiceDNSName(
corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: svcRef.Namespace, Name: svcRef.Name}},
)
for _, n := range names {
svcNames[n] = struct{}{}
}
Expand Down
20 changes: 5 additions & 15 deletions pkg/controller/webhook/reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,38 +25,28 @@ type Params struct {

// ReconcileResources reconciles the certificates used by the webhook client and the webhook server.
// It also returns the duration after which a certificate rotation should be scheduled.
func (w *Params) ReconcileResources(ctx context.Context, clientset kubernetes.Interface) error {
func (w *Params) ReconcileResources(ctx context.Context, clientset kubernetes.Interface, webhookConfiguration AdmissionControllerInterface) error {
// retrieve current webhook server cert secret
webhookServerSecret, err := clientset.CoreV1().Secrets(w.Namespace).Get(ctx, w.SecretName, metav1.GetOptions{})
if err != nil {
// 404 is still considered as an error, webhook secret is expected to be created before the operator is started
return err
}

// retrieve the current webhook configuration
webhookConfiguration, err := clientset.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
if err != nil {
// 404 is also considered as an error, webhook configuration is expected to be created before the operator is started
return err
}

// check if we need to renew the certificates used in the resources
if w.shouldRenewCertificates(webhookServerSecret, webhookConfiguration) {
if w.shouldRenewCertificates(webhookServerSecret, webhookConfiguration.webhooks()) {
log.Info(
"Creating new webhook certificates",
"webhook", webhookConfiguration.Name,
"webhook", w.Name,
"secret_namespace", webhookServerSecret.Namespace,
"secret_name", webhookServerSecret.Name,
)
newCertificates, err := w.newCertificates(webhookConfiguration)
newCertificates, err := w.newCertificates(webhookConfiguration.services())
if err != nil {
return err
}
// update the webhook configuration
for i := range webhookConfiguration.Webhooks {
webhookConfiguration.Webhooks[i].ClientConfig.CABundle = newCertificates.caCert
}
if _, err := clientset.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Update(ctx, webhookConfiguration, metav1.UpdateOptions{}); err != nil {
if err := webhookConfiguration.updateCABundle(newCertificates.caCert); err != nil {
return err
}

Expand Down
33 changes: 24 additions & 9 deletions pkg/controller/webhook/reconcile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"testing"

"github.com/stretchr/testify/assert"
"k8s.io/api/admissionregistration/v1beta1"
v1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
Expand All @@ -38,22 +38,37 @@ func TestParams_ReconcileResources(t *testing.T) {
Name: "elastic-webhook-server-cert",
},
},
&v1beta1.ValidatingWebhookConfiguration{
&v1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "elastic-webhook.k8s.elastic.co",
},
Webhooks: []v1beta1.ValidatingWebhook{
Webhooks: []v1.ValidatingWebhook{
{
Name: "elastic-es-validation-v1.k8s.elastic.co",
ClientConfig: v1beta1.WebhookClientConfig{
Service: &v1beta1.ServiceReference{Name: "elastic-webhook-server", Namespace: "elastic-system"},
ClientConfig: v1.WebhookClientConfig{
Service: &v1.ServiceReference{Name: "elastic-webhook-server", Namespace: "elastic-system"},
},
},
},
},
)

if err := w.ReconcileResources(context.Background(), clientset); err != nil {
clientset.Resources = []*metav1.APIResourceList{
{
GroupVersion: "admissionregistration.k8s.io/v1",
APIResources: []metav1.APIResource{
{Name: "admissionregistration.k8s.io", Namespaced: false, Kind: "APIGroup", Group: "admissionregistration.k8s.io", Version: "v1"},
},
},
}

// retrieve the current webhook configuration interface
wh, err := w.NewAdmissionControllerInterface(context.Background(), clientset)
if err != nil {
t.Errorf("Params.NewAdmissionControllerInterface() error = %v", err)
}

if err := w.ReconcileResources(context.Background(), clientset, wh); err != nil {
t.Errorf("Params.ReconcileResources() error = %v", err)
}

Expand All @@ -65,7 +80,7 @@ func TestParams_ReconcileResources(t *testing.T) {
assert.Equal(t, 2, len(webhookServerSecret.Data))

// retrieve the current webhook configuration
webhookConfiguration, err := clientset.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
webhookConfiguration, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
assert.NoError(t, err)
caBundle := webhookConfiguration.Webhooks[0].ClientConfig.CABundle
assert.True(t, len(caBundle) > 0)
Expand All @@ -81,7 +96,7 @@ func TestParams_ReconcileResources(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 0, len(webhookServerSecret.Data))

if err := w.ReconcileResources(ctx, clientset); err != nil {
if err := w.ReconcileResources(ctx, clientset, wh); err != nil {
t.Errorf("Params.ReconcileResources() error = %v", err)
}

Expand All @@ -90,7 +105,7 @@ func TestParams_ReconcileResources(t *testing.T) {
assert.Equal(t, 2, len(webhookServerSecret.Data))

// retrieve the new ca
webhookConfiguration, err = clientset.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
webhookConfiguration, err = clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, w.Name, metav1.GetOptions{})
assert.NoError(t, err)
caBundle = webhookConfiguration.Webhooks[0].ClientConfig.CABundle
// Check again that the cert in the secret has been signed by the caBundle
Expand Down
Loading

0 comments on commit db29820

Please sign in to comment.