Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

certrotationcontroller: use minutes instead of days when FeatureShortCertRotation is enabled #817

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,5 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)

replace github.com/openshift/api => github.com/vrutkovs/api v0.0.0-20241107084801-3929a3da5d2a
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,6 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA
github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
github.com/openshift/api v0.0.0-20241001152557-e415140e5d5f h1:ya1OmyZm3LIIxI3U9VE9Nyx3ehCHgBwxyFUPflYPWls=
github.com/openshift/api v0.0.0-20241001152557-e415140e5d5f/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
github.com/openshift/build-machinery-go v0.0.0-20240613134303-8359781da660 h1:F0zE2bmdVvaEd18VXuGYQdJJ1FYJu4MIDW9PYZWc9No=
github.com/openshift/build-machinery-go v0.0.0-20240613134303-8359781da660/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE=
github.com/openshift/client-go v0.0.0-20241001162912-da6d55e4611f h1:FRc0bVNWprihWS0GqQWzb3dY4dkCwpOP3mDw5NwSoR4=
Expand Down Expand Up @@ -212,6 +210,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
github.com/vrutkovs/api v0.0.0-20241107084801-3929a3da5d2a h1:GxSAOagJfIOicyVPtKMaYrJNuXM2g2I46CAEE8PFTxM=
github.com/vrutkovs/api v0.0.0-20241107084801-3929a3da5d2a/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
Expand Down
29 changes: 22 additions & 7 deletions pkg/cmd/recoverycontroller/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@ package recoverycontroller
import (
"context"
"fmt"
"time"

operatorv1 "github.com/openshift/api/operator/v1"
configeversionedclient "github.com/openshift/client-go/config/clientset/versioned"
configexternalinformers "github.com/openshift/client-go/config/informers/externalversions"
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator"
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/certrotationcontroller"
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/operatorclient"
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/version"
"github.com/openshift/library-go/pkg/controller/controllercmd"
"github.com/openshift/library-go/pkg/operator/certrotation"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/openshift/library-go/pkg/operator/genericoperatorclient"
"github.com/openshift/library-go/pkg/operator/status"
"github.com/openshift/library-go/pkg/operator/v1helpers"
"github.com/spf13/cobra"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -91,21 +95,28 @@ func (o *Options) Run(ctx context.Context) error {
return err
}

certRotationScale, err := certrotation.GetCertRotationScale(ctx, kubeClient, operatorclient.GlobalUserSpecifiedConfigNamespace)
configClient, err := configeversionedclient.NewForConfig(o.controllerContext.KubeConfig)
if err != nil {
return err
return fmt.Errorf("failed to create config client: %w", err)
}
configInformers := configexternalinformers.NewSharedInformerFactory(configClient, 10*time.Minute)
desiredVersion := status.VersionForOperatorFromEnv()
missingVersion := "0.0.1-snapshot"
featureGateAccessor := featuregates.NewFeatureGateAccess(
desiredVersion, missingVersion,
configInformers.Config().V1().ClusterVersions(), configInformers.Config().V1().FeatureGates(),
o.controllerContext.EventRecorder,
)
go featureGateAccessor.Run(ctx)
go configInformers.Start(ctx.Done())

certRotationController, err := certrotationcontroller.NewCertRotationControllerOnlyWhenExpired(
v1helpers.CachedSecretGetter(kubeClient.CoreV1(), kubeInformersForNamespaces),
v1helpers.CachedConfigMapGetter(kubeClient.CoreV1(), kubeInformersForNamespaces),
operatorClient,
kubeInformersForNamespaces,
o.controllerContext.EventRecorder,
// this is weird, but when we turn down rotation in CI, we go fast enough that kubelets and kas are racing to observe the new signer before the signer is used.
// we need to establish some kind of delay or back pressure to prevent the rollout. This ensures we don't trigger kas restart
// during e2e tests for now.
certRotationScale*8,
featureGateAccessor,
)
if err != nil {
return err
Expand Down Expand Up @@ -134,6 +145,10 @@ func (o *Options) Run(ctx context.Context) error {
csrController.Run(ctx)
}()

go func() {
featureGateAccessor.Run(ctx)
}()

<-ctx.Done()

return nil
Expand Down
37 changes: 21 additions & 16 deletions pkg/operator/certrotationcontroller/certrotationcontroller.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,19 @@ package certrotationcontroller

import (
"context"
"fmt"
"time"

"k8s.io/klog/v2"

corev1client "k8s.io/client-go/kubernetes/typed/core/v1"

"github.com/openshift/library-go/pkg/operator/certrotation"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"

features "github.com/openshift/api/features"
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/operatorclient"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
)

// defaultRotationDay is the default rotation base for all cert rotation operations.
Expand All @@ -29,15 +30,15 @@ func NewCertRotationController(
operatorClient v1helpers.StaticPodOperatorClient,
kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
eventRecorder events.Recorder,
day time.Duration,
featureGateAccessor featuregates.FeatureGateAccess,
) (*CertRotationController, error) {
return newCertRotationController(
secretsGetter,
configMapsGetter,
operatorClient,
kubeInformersForNamespaces,
eventRecorder,
day,
featureGateAccessor,
false,
)
}
Expand All @@ -48,15 +49,15 @@ func NewCertRotationControllerOnlyWhenExpired(
operatorClient v1helpers.StaticPodOperatorClient,
kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
eventRecorder events.Recorder,
day time.Duration,
featureGateAccessor featuregates.FeatureGateAccess,
) (*CertRotationController, error) {
return newCertRotationController(
secretsGetter,
configMapsGetter,
operatorClient,
kubeInformersForNamespaces,
eventRecorder,
day,
featureGateAccessor,
true,
)
}
Expand All @@ -67,16 +68,20 @@ func newCertRotationController(
operatorClient v1helpers.StaticPodOperatorClient,
kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
eventRecorder events.Recorder,
day time.Duration,
featureGateAccessor featuregates.FeatureGateAccess,
refreshOnlyWhenExpired bool,
) (*CertRotationController, error) {
ret := &CertRotationController{}

rotationDay := defaultRotationDay
if day != time.Duration(0) {
rotationDay = day
klog.Warningf("!!! UNSUPPORTED VALUE SET !!!")
klog.Warningf("Certificate rotation base set to %q", rotationDay)
monthPeriod := time.Hour * 24 * 30

featureGates, err := featureGateAccessor.CurrentFeatureGates()
if err != nil {
return nil, fmt.Errorf("unable to get FeatureGates: %w", err)
}

if featureGates.Enabled(features.FeatureShortCertRotation) {
monthPeriod = 90 * time.Minute
}

certRotator := certrotation.NewCertRotationController(
Expand All @@ -88,8 +93,8 @@ func newCertRotationController(
AdditionalAnnotations: certrotation.AdditionalAnnotations{
JiraComponent: "kube-controller-manager",
},
Validity: 60 * rotationDay,
Refresh: 30 * rotationDay,
Validity: monthPeriod * 2,
Refresh: monthPeriod,
RefreshOnlyWhenExpired: refreshOnlyWhenExpired,
Informer: kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets(),
Lister: kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets().Lister(),
Expand All @@ -113,8 +118,8 @@ func newCertRotationController(
AdditionalAnnotations: certrotation.AdditionalAnnotations{
JiraComponent: "kube-controller-manager",
},
Validity: 30 * rotationDay,
Refresh: 15 * rotationDay,
Validity: monthPeriod,
Refresh: monthPeriod / 2,
RefreshOnlyWhenExpired: refreshOnlyWhenExpired,
CertCreator: &certrotation.SignerRotation{
SignerName: "kube-csr-signer",
Expand Down
11 changes: 1 addition & 10 deletions pkg/operator/starter.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/resourcesynccontroller"
"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/targetconfigcontroller"
"github.com/openshift/library-go/pkg/controller/controllercmd"
"github.com/openshift/library-go/pkg/operator/certrotation"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/openshift/library-go/pkg/operator/genericoperatorclient"
"github.com/openshift/library-go/pkg/operator/latencyprofilecontroller"
Expand Down Expand Up @@ -260,21 +259,13 @@ func RunOperator(ctx context.Context, cc *controllercmd.ControllerContext) error
cc.EventRecorder,
)

certRotationScale, err := certrotation.GetCertRotationScale(ctx, kubeClient, operatorclient.GlobalUserSpecifiedConfigNamespace)
if err != nil {
return err
}

certRotationController, err := certrotationcontroller.NewCertRotationController(
v1helpers.CachedSecretGetter(kubeClient.CoreV1(), kubeInformersForNamespaces),
v1helpers.CachedConfigMapGetter(kubeClient.CoreV1(), kubeInformersForNamespaces),
operatorClient,
kubeInformersForNamespaces,
cc.EventRecorder,
// this is weird, but when we turn down rotation in CI, we go fast enough that kubelets and kas are racing to observe the new signer before the signer is used.
// we need to establish some kind of delay or back pressure to prevent the rollout. This ensures we don't trigger kas restart
// during e2e tests for now.
certRotationScale*8,
featureGateAccessor,
)
if err != nil {
return err
Expand Down
26 changes: 20 additions & 6 deletions vendor/github.com/openshift/api/config/v1/types_infrastructure.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 19 additions & 0 deletions vendor/github.com/openshift/api/config/v1/types_node.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading