Skip to content

Commit

Permalink
Allow control plane provider to set endpoint
Browse files Browse the repository at this point in the history 
Signed-off-by: Vince Prignano <vince@prigna.com>
  • Loading branch information
@vincepri
vincepri committed May 28, 2024
1 parent 2568aa2 commit 686a919
Show file tree
Hide file tree
Showing 4 changed files with 278 additions and 5 deletions.
14 changes: 10 additions & 4 deletions docs/book/src/developer/architecture/controllers/control-plane.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@ Kubernetes control plane consisting of the following services:

The Cluster controller will set an OwnerReference on the Control Plane. The Control Plane controller should normally take no action during reconciliation until it sees the ownerReference.

A Control Plane controller implementation should exit reconciliation until it sees `cluster.spec.controlPlaneEndpoint` populated.
A Control Plane controller implementation must either supply a controlPlaneEndpoint,
or rely on `spec.controlPlaneEndpoint` in its parent [Cluster](./cluster.md) object.

The Cluster controller bubbles up `status.ready` into `status.controlPlaneReady` and `status.initialized` into a `controlPlaneInitialized` condition from the Control Plane CR.
If an endpoint is not provider, the implementer should exit reconciliation until it sees `cluster.spec.controlPlaneEndpoint` populated.

A Control Plane controller can optionally provide a `controlPlaneEndpoint`

The `ImplementationControlPlane` *must* rely on the existence of
`status.controlplaneEndpoint` in its parent [Cluster](./cluster.md) object.
The Cluster controller bubbles up `status.ready` into `status.controlPlaneReady` and `status.initialized` into a `controlPlaneInitialized` condition from the Control Plane CR.

### CRD contracts

Expand Down Expand Up @@ -110,6 +112,10 @@ documentation][scale].
deletion. A duration of 0 will retry deletion indefinitely. It defaults to 10 seconds on the
Machine.

#### Optional `spec` fields for implementations providing endpoints

The `ImplementationControlPlane` object may provide a `spec.controlPlaneEndpoint` field to inform the Cluster controller where the endpoint is located.

#### Required `status` fields

The `ImplementationControlPlane` object **must** have a `status` object.
Expand Down
1 change: 1 addition & 0 deletions docs/proposals/20230407-flexible-managed-k8s-endpoints.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ More specifically we would like to introduce first class support for two scenari

- Permit omitting the `<Infra>Cluster` entirely, thus making it simpler to use with Cluster API all the Managed Kubernetes implementations which do not require any additional Kubernetes Cluster Infrastructure (network settings, security groups, etc) on top of what is provided out of the box by the managed Kubernetes primitive offered by a Cloud provider.
- Allow the `ControlPlane Provider` component to take ownership of the responsibility of creating the control plane endpoint, thus making it simpler to use with Cluster API all the Managed Kubernetes implementations which are taking care out of the box of this piece of Cluster Infrastructure.
- Note: In May 2024 [this pull request](https://github.com/kubernetes-sigs/cluster-api/pull/10667) added the ability for the control plane provider to provide the endpoint the same way the infrastructure reference would.

The above capabilities can be used alone or in combination depending on the requirements of a specific Managed Kubernetes or on the specific architecture/set of Cloud components being implemented.

Expand Down
8 changes: 8 additions & 0 deletions internal/controllers/cluster/cluster_controller_phases.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,14 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster
}
}

// Get and parse Spec.ControlPlaneEndpoint field from the infrastructure provider.
if !cluster.Spec.ControlPlaneEndpoint.IsValid() {
if err := util.UnstructuredUnmarshalField(controlPlaneConfig, &cluster.Spec.ControlPlaneEndpoint, "spec", "controlPlaneEndpoint"); err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Spec.ControlPlaneEndpoint from control plane provider for Cluster %q in namespace %q",
cluster.Name, cluster.Namespace)
}
}

return ctrl.Result{}, nil
}

Expand Down
260 changes: 259 additions & 1 deletion internal/controllers/cluster/cluster_controller_phases_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capierrors "sigs.k8s.io/cluster-api/errors"
"sigs.k8s.io/cluster-api/internal/test/builder"
"sigs.k8s.io/cluster-api/util/conditions"
)

func TestClusterReconcilePhases(t *testing.T) {
Expand All @@ -56,13 +57,30 @@ func TestClusterReconcilePhases(t *testing.T) {
},
},
}
clusterNoEndpoint := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
Namespace: "test-namespace",
},
Status: clusterv1.ClusterStatus{
InfrastructureReady: true,
},
Spec: clusterv1.ClusterSpec{
InfrastructureRef: &corev1.ObjectReference{
APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
Kind: "GenericInfrastructureMachine",
Name: "test",
},
},
}

tests := []struct {
name string
cluster *clusterv1.Cluster
infraRef map[string]interface{}
expectErr bool
expectResult ctrl.Result
check func(g *GomegaWithT, in *clusterv1.Cluster)
}{
{
name: "returns no error if infrastructure ref is nil",
Expand Down Expand Up @@ -104,7 +122,7 @@ func TestClusterReconcilePhases(t *testing.T) {
expectErr: false,
},
{
name: "returns error if infrastructure has the paused annotation",
name: "returns no error if infrastructure has the paused annotation",
cluster: cluster,
infraRef: map[string]interface{}{
"kind": "GenericInfrastructureMachine",
Expand All @@ -119,6 +137,50 @@ func TestClusterReconcilePhases(t *testing.T) {
},
expectErr: false,
},
{
name: "returns an error if the control plane endpoint is not yet set",
cluster: clusterNoEndpoint,
infraRef: map[string]interface{}{
"kind": "GenericInfrastructureMachine",
"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"deletionTimestamp": "sometime",
},
"status": map[string]interface{}{
"ready": true,
},
},
expectErr: true,
},
{
name: "should propagate the control plane endpoint once set",
cluster: clusterNoEndpoint,
infraRef: map[string]interface{}{
"kind": "GenericInfrastructureMachine",
"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"deletionTimestamp": "sometime",
},
"spec": map[string]interface{}{
"controlPlaneEndpoint": map[string]interface{}{
"host": "example.com",
"port": int64(6443),
},
},
"status": map[string]interface{}{
"ready": true,
},
},
expectErr: false,
check: func(g *GomegaWithT, in *clusterv1.Cluster) {
g.Expect(in.Spec.ControlPlaneEndpoint.Host).To(Equal("example.com"))
g.Expect(in.Spec.ControlPlaneEndpoint.Port).To(BeEquivalentTo(6443))
},
},
}

for _, tt := range tests {
Expand Down Expand Up @@ -149,6 +211,202 @@ func TestClusterReconcilePhases(t *testing.T) {
} else {
g.Expect(err).ToNot(HaveOccurred())
}

if tt.check != nil {
tt.check(g, tt.cluster)
}
})
}
})

t.Run("reconcile control plane ref", func(t *testing.T) {
cluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
Namespace: "test-namespace",
},
Status: clusterv1.ClusterStatus{
InfrastructureReady: true,
},
Spec: clusterv1.ClusterSpec{
ControlPlaneEndpoint: clusterv1.APIEndpoint{
Host: "1.2.3.4",
Port: 8443,
},
ControlPlaneRef: &corev1.ObjectReference{
APIVersion: "controlplane.cluster.x-k8s.io/v1beta1",
Kind: "GenericControlPlane",
Name: "test",
},
},
}
clusterNoEndpoint := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
Namespace: "test-namespace",
},
Status: clusterv1.ClusterStatus{
InfrastructureReady: true,
},
Spec: clusterv1.ClusterSpec{
ControlPlaneRef: &corev1.ObjectReference{
APIVersion: "controlplane.cluster.x-k8s.io/v1beta1",
Kind: "GenericControlPlane",
Name: "test",
},
},
}

tests := []struct {
name string
cluster *clusterv1.Cluster
cpRef map[string]interface{}
expectErr bool
expectResult ctrl.Result
check func(g *GomegaWithT, in *clusterv1.Cluster)
}{
{
name: "returns no error if control plane ref is nil",
cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "test-cluster", Namespace: "test-namespace"}},
expectErr: false,
},
{
name: "returns error if unable to reconcile control plane ref",
cluster: cluster,
expectErr: false,
expectResult: ctrl.Result{RequeueAfter: 30 * time.Second},
},
{
name: "returns no error if infra config is marked for deletion",
cluster: cluster,
cpRef: map[string]interface{}{
"kind": "GenericControlPlane",
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"deletionTimestamp": "sometime",
},
},
expectErr: false,
},
{
name: "returns no error if infrastructure has the paused annotation",
cluster: cluster,
cpRef: map[string]interface{}{
"kind": "GenericControlPlane",
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"annotations": map[string]interface{}{
"cluster.x-k8s.io/paused": "true",
},
},
},
expectErr: false,
},
{
name: "returns an error if the control plane endpoint is not yet set",
cluster: clusterNoEndpoint,
cpRef: map[string]interface{}{
"kind": "GenericControlPlane",
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"deletionTimestamp": "sometime",
},
"status": map[string]interface{}{
"ready": true,
},
},
expectErr: true,
},
{
name: "should propagate the control plane endpoint if set",
cluster: clusterNoEndpoint,
cpRef: map[string]interface{}{
"kind": "GenericControlPlane",
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"deletionTimestamp": "sometime",
},
"spec": map[string]interface{}{
"controlPlaneEndpoint": map[string]interface{}{
"host": "example.com",
"port": int64(6443),
},
},
"status": map[string]interface{}{
"ready": true,
},
},
expectErr: false,
check: func(g *GomegaWithT, in *clusterv1.Cluster) {
g.Expect(in.Spec.ControlPlaneEndpoint.Host).To(Equal("example.com"))
g.Expect(in.Spec.ControlPlaneEndpoint.Port).To(BeEquivalentTo(6443))
},
},
{
name: "should propagate the initialized and ready conditions",
cluster: clusterNoEndpoint,
cpRef: map[string]interface{}{
"kind": "GenericControlPlane",
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta1",
"metadata": map[string]interface{}{
"name": "test",
"namespace": "test-namespace",
"deletionTimestamp": "sometime",
},
"spec": map[string]interface{}{},
"status": map[string]interface{}{
"ready": true,
"initialized": true,
},
},
expectErr: false,
check: func(g *GomegaWithT, in *clusterv1.Cluster) {
g.Expect(conditions.IsTrue(in, clusterv1.ControlPlaneReadyCondition)).To(BeTrue())
g.Expect(conditions.IsTrue(in, clusterv1.ControlPlaneInitializedCondition)).To(BeTrue())
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)

var c client.Client
if tt.cpRef != nil {
infraConfig := &unstructured.Unstructured{Object: tt.cpRef}
c = fake.NewClientBuilder().
WithObjects(builder.GenericControlPlaneCRD.DeepCopy(), tt.cluster, infraConfig).
Build()
} else {
c = fake.NewClientBuilder().
WithObjects(builder.GenericControlPlaneCRD.DeepCopy(), tt.cluster).
Build()
}
r := &Reconciler{
Client: c,
UnstructuredCachingClient: c,
recorder: record.NewFakeRecorder(32),
}

res, err := r.reconcileControlPlane(ctx, tt.cluster)
g.Expect(res).To(BeComparableTo(tt.expectResult))
if tt.expectErr {
g.Expect(err).To(HaveOccurred())
} else {
g.Expect(err).ToNot(HaveOccurred())
}

if tt.check != nil {
tt.check(g, tt.cluster)
}
})
}
})
Expand Down

0 comments on commit 686a919

Please sign in to comment.