Skip to content

Commit

Permalink
CAPX clusterclass support (#344)
Browse files Browse the repository at this point in the history
* capx clusterclass support

have kept the go/v3 kuberbuilder scffolding as is so reduce the code
churn

* updated yamls and their kustomize files. also review comments incorporated

* made user-ca-bundle cluster specific and separated cluster and clusterclass

* fixed typos

* updated eviction policy and master merge template fixes

* merged ccm related changed with the new yaml templates

* added first jsonPatches in clusterclass template and cluster topology

* added controlPlaneEnpoint variable in clusterclass

* added pc endpoint details in patches and variables

* fixed nct

* added nmt patches for cp and md

* reverted removal of ccm ca bundle config map and added test

* fixed makefile and docs with correct commands

* now using infrav1 instead of kubebuilder default for latest version

* update the docs

* e2e test fixes

* removed sample file for nutanixclustertemplate

* added cni-crs related patch

* updated test to match the fields

* separated dev test makefile commands into separate file

this is done to improve maintainability

* yaml location

* updated name of cm to make the test pass

This is required as currently ccm-update.yaml vars are not getting
replaced thru code.
  • Loading branch information
deepakm-ntnx authored Jan 15, 2024
1 parent ba2b194 commit d5cf036
Show file tree
Hide file tree
Showing 72 changed files with 3,123 additions and 242 deletions.
45 changes: 15 additions & 30 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -181,9 +181,6 @@ USE_EXISTING_CLUSTER ?= false
GINKGO_NOCOLOR ?= false
FLAVOR ?= e2e

TEST_NAMESPACE=capx-test-ns
TEST_CLUSTER_NAME=mycluster

# set ginkgo focus flags, if any
ifneq ($(strip $(GINKGO_FOCUS)),)
_FOCUS_ARGS := $(foreach arg,$(strip $(GINKGO_FOCUS)),--focus="$(arg)")
Expand Down Expand Up @@ -328,6 +325,9 @@ cluster-e2e-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-topology.yaml

cluster-e2e-templates-no-kubeproxy: $(KUSTOMIZE) ##Generate cluster templates without kubeproxy
# v1alpha4
Expand All @@ -346,10 +346,15 @@ cluster-e2e-templates-no-kubeproxy: $(KUSTOMIZE) ##Generate cluster templates wi
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml
$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-topology.yaml

cluster-templates: $(KUSTOMIZE) ## Generate cluster templates for all flavors
$(KUSTOMIZE) build $(TEMPLATES_DIR)/base > $(TEMPLATES_DIR)/cluster-template.yaml
$(KUSTOMIZE) build $(TEMPLATES_DIR)/csi > $(TEMPLATES_DIR)/cluster-template-csi.yaml
$(KUSTOMIZE) build $(TEMPLATES_DIR)/clusterclass > $(TEMPLATES_DIR)/cluster-template-clusterclass.yaml
$(KUSTOMIZE) build $(TEMPLATES_DIR)/topology > $(TEMPLATES_DIR)/cluster-template-topology.yaml

##@ Testing

Expand Down Expand Up @@ -387,33 +392,6 @@ ifeq ($(EXPORT_RESULT), true)
gocov convert profile.cov | gocov-xml > coverage.xml
endif

.PHONY: test-clusterctl-create
test-clusterctl-create: $(CLUSTERCTL) ## Run the tests using clusterctl
$(CLUSTERCTL) version
$(CLUSTERCTL) config repositories | grep nutanix
$(CLUSTERCTL) generate cluster ${TEST_CLUSTER_NAME} -i nutanix:${LOCAL_PROVIDER_VERSION} --list-variables -v 10
$(CLUSTERCTL) generate cluster ${TEST_CLUSTER_NAME} -i nutanix:${LOCAL_PROVIDER_VERSION} --target-namespace ${TEST_NAMESPACE} -v 10 > ./cluster.yaml
kubectl create ns $(TEST_NAMESPACE) || true
kubectl apply -f ./cluster.yaml -n $(TEST_NAMESPACE)

.PHONY: test-clusterctl-delete
test-clusterctl-delete: ## Delete clusterctl created cluster
kubectl -n ${TEST_NAMESPACE} delete cluster ${TEST_CLUSTER_NAME}

.PHONY: test-kubectl-bootstrap
test-kubectl-bootstrap: ## Run kubectl queries to get all capx management/bootstrap related objects
kubectl get ns
kubectl get all --all-namespaces
kubectl -n capx-system get all
kubectl -n $(TEST_NAMESPACE) get Cluster,NutanixCluster,Machine,NutanixMachine,KubeAdmControlPlane,MachineHealthCheck,nodes
kubectl -n capx-system get pod

.PHONY: test-kubectl-workload
test-kubectl-workload: ## Run kubectl queries to get all capx workload related objects
kubectl -n $(TEST_NAMESPACE) get secret
kubectl -n ${TEST_NAMESPACE} get secret ${TEST_CLUSTER_NAME}-kubeconfig -o json | jq -r .data.value | base64 --decode > ${TEST_CLUSTER_NAME}.workload.kubeconfig
kubectl --kubeconfig ./${TEST_CLUSTER_NAME}.workload.kubeconfig get nodes,ns

.PHONY: ginkgo-help
ginkgo-help:
$(GINKGO) help run
Expand Down Expand Up @@ -631,3 +609,10 @@ verify-manifests: manifests ## Verify generated manifests are up to date
clean: ## Clean the build and test artifacts
rm -rf $(ARTIFACTS) $(BIN_DIR)

## --------------------------------------
## Developer local tests
## --------------------------------------

##@ Test Dev Cluster with and without topology
include test-cluster-without-topology.mk
include test-cluster-with-topology.mk
12 changes: 12 additions & 0 deletions PROJECT
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# Code generated by tool. DO NOT EDIT.
# This file is used to track the info used to scaffold your project
# and allow the plugins properly work.
# More info: https://book.kubebuilder.io/reference/project-config.html
domain: cluster.x-k8s.io
layout:
- go.kubebuilder.io/v3
Expand Down Expand Up @@ -30,4 +34,12 @@ resources:
kind: NutanixMachineTemplate
path: github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1alpha4
version: v1alpha4
- api:
crdVersion: v1
namespaced: true
domain: cluster.x-k8s.io
group: infrastructure
kind: NutanixClusterTemplate
path: github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1
version: v1beta1
version: "3"
55 changes: 55 additions & 0 deletions api/v1beta1/nutanixclustertemplate_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
Copyright 2022 Nutanix
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// NutanixClusterTemplateSpec defines the desired state of NutanixClusterTemplate
type NutanixClusterTemplateSpec struct {
Template NutanixClusterTemplateResource `json:"template"`
}

//+kubebuilder:object:root=true
//+kubebuilder:resource:categories=cluster-api

// NutanixClusterTemplate is the Schema for the nutanixclustertemplates API
type NutanixClusterTemplate struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec NutanixClusterTemplateSpec `json:"spec,omitempty"`
}

//+kubebuilder:object:root=true

// NutanixClusterTemplateList contains a list of NutanixClusterTemplate
type NutanixClusterTemplateList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []NutanixClusterTemplate `json:"items"`
}

func init() {
SchemeBuilder.Register(&NutanixClusterTemplate{}, &NutanixClusterTemplateList{})
}

// NutanixClusterTemplateResource describes the data needed to create a NutanixCluster from a template.
type NutanixClusterTemplateResource struct {
Spec NutanixClusterSpec `json:"spec"`
}
90 changes: 90 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions clusterctl.yaml.tmpl
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
CLUSTERCTL_LOG_LEVEL: 10
EXP_CLUSTER_RESOURCE_SET: "true"
CLUSTER_TOPOLOGY: "true"

NUTANIX_ENDPOINT: "" # IP or FQDN of Prism Central
NUTANIX_USER: "" # Prism Central user
Expand All @@ -17,6 +18,8 @@ WORKER_MACHINE_COUNT: 2 # Number of Machines in the worker node
KUBEVIP_LB_ENABLE: "false" # See https://kube-vip.io/docs/about/architecture/?query=lb_enable#control-plane-load-balancing
KUBEVIP_SVC_ENABLE: "false" # See https://kube-vip.io/docs/about/architecture/?query=lb_enable#kubernetes-service-load-balancing

CLUSTER_CLASS_NAME: "" # ClusterClass Name if any

providers:
# add a custom provider
- name: "nutanix"
Expand Down
Loading

0 comments on commit d5cf036

Please sign in to comment.