From 2feef4f3cdcb170b779839b7b2f774bf114efe65 Mon Sep 17 00:00:00 2001 From: nasusoba Date: Tue, 23 Apr 2024 15:07:03 +0800 Subject: [PATCH] add a doc for tilt setup Signed-off-by: nasusoba --- docs/tilt-setup.md | 127 ++++++++++++++++++ test/e2e/README.md | 4 +- .../cluster-template.yaml | 9 +- 3 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 docs/tilt-setup.md diff --git a/docs/tilt-setup.md b/docs/tilt-setup.md new file mode 100644 index 00000000..01973c33 --- /dev/null +++ b/docs/tilt-setup.md @@ -0,0 +1,127 @@ +# Developing Cluster API K3s with Tilt +This document describes how to use kind and [Tilt][tilt] for a simplified workflow that offers easy deployments and rapid iterative builds. It helps you setup a local development environment with a kind cluster as the management cluster and [CAPD][capd] as the infrastructure provider. + +Also, visit the [Cluster API documentation on Tilt][cluster_api_tilt] for more information on how to set up your development environment. + +[tilt]: https://tilt.dev +[cluster_api_tilt]: https://cluster-api.sigs.k8s.io/developer/tilt.html +[capd]: https://github.com/kubernetes-sigs/cluster-api/tree/main/test/infrastructure/docker + +## Getting started + +1. You need to follow the [Prerequisites in Cluster API documentation on Tilt](https://cluster-api.sigs.k8s.io/developer/tilt.html#prerequisites) to have everything installed. + +2. Get the source for cluster-api repo + + ```bash + git clone https://github.com/kubernetes-sigs/cluster-api.git + ``` + + In this guide, the `cluster-api` repo is put in the same directory as the `cluster-api-k3s` repo. It is fine if you put it in a different directory, but you might need to change the paths in the `tilt-settings.yaml`. + + ```bash + $ ls + cluster-api cluster-api-k3s + ``` + +3. Create the management cluster + + ```bash + cd cluster-api + ./hack/kind-install-for-capd.sh + ``` + +4. Create the `tilt-settings.yaml` file and place it in your local copy of `cluster-api`. Here is an example: + ```yaml + # refer to https://cluster-api.sigs.k8s.io/developer/tilt.html for documentation + allowed_contexts: + - kind-capi-test + trigger_mode: manual # set to auto to enable auto-rebuilding + default_registry: '' # empty means use local registry + provider_repos: + - ../cluster-api-k3s # load k3s as a provider, change to a different path if needed + enable_providers: + - docker + - k3s-bootstrap + - k3s-control-plane + deploy_observability: + - visualizer + kustomize_substitutions: + # enable some experiment features + CLUSTER_TOPOLOGY: "true" + EXP_MACHINE_POOL: "true" + EXP_CLUSTER_RESOURCE_SET: "true" + EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" + EXP_RUNTIME_SDK: "true" + # add variables for workload cluster template + KUBERNETES_VERSION: "v1.28.6+k3s2" + KIND_IMAGE_VERSION: "v1.28.0" + WORKER_MACHINE_COUNT: "1" + CONTROL_PLANE_MACHINE_COUNT: "1" + # Note: kustomize substitutions expects the values to be strings. This can be achieved by wrapping the values in quotation marks. + # also, can use this to provide credentials + kind_cluster_name: capi-test + extra_args: + # add extra arguments when launching the binary + k3s-bootstrap: + - --enable-leader-election=false + k3s-control-plane: + - --enable-leader-election=false + debug: + # enable delve for debugging + docker: + continue: true # change to false if you need the service to be running after the delve has been connected + port: 30000 + profiler_port: 30001 + metrics_port: 30002 + core: + continue: true + port: 31000 + profiler_port: 31001 + metrics_port: 31002 + k3s-bootstrap: + continue: true + port: 32000 + k3s-control-plane: + continue: true + port: 33000 + template_dirs: + # add template for fast workload cluster creation, change to a different path if needed + # you could also add more templates + k3s-bootstrap: + - ../cluster-api-k3s/test/e2e/data/infrastructure-docker + ``` +5. Run `tilt` in the `cluster-api` directory + + ```bash + tilt up + ``` + +6. Create a cluster using the `tilt` UI: if you have `template_dirs` set in the `tilt-settings.yaml`, you will find the `CABP3.templates` resource and you could create a cluster with one click! + +## Debugging + +If you would like to debug CAPI K3s (or core CAPI / another provider) you can run the provider with delve. This will then allow you to attach to delve and debug. Please check the `debug` part in `tilt-settings.yaml`. + +For vscode, you can use a launch configuration like this (please change the `port` to the one you have set in `tilt-settings.yaml`): +```json +{ + "name": "K3s Bootstrap Controller", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "", + "port": 32000, + "host": "127.0.0.1", + "showLog": true, + "trace": "log", + "logOutput": "rpc" +} +``` +## Clean up +Before deleting the kind cluster, make sure you delete all the workload clusters. +```bash +kubectl delete cluster +tilt up (ctrl-c) +kind delete cluster +``` \ No newline at end of file diff --git a/test/e2e/README.md b/test/e2e/README.md index 138f1e97..f2d56da7 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -10,9 +10,9 @@ make docker-build-e2e # should be run everytime you change the controller code make test-e2e # run all e2e tests ``` ### Run a specific e2e test -To run a specific e2e test, such as "PR-Blocking", use the `GINKGO_FOCUS` environment variable as shown below: +To run a specific e2e test, such as `[PR-Blocking]`, use the `GINKGO_FOCUS` environment variable as shown below: ```shell -make GINKGO_FOCUS="\\[PR-Blocking\\]" test-e2e # only run the "PR-Blocking" e2e test +make GINKGO_FOCUS="\\[PR-Blocking\\]" test-e2e # only run e2e test with `[PR-Blocking]` in its spec name ``` ### Run the e2e test with tilt It is quite useful to run the e2e test with [tilt](https://cluster-api.sigs.k8s.io/developer/tilt), so that you will not need to rebuild docker image with `make docker-build-e2e` everytime. Also you will not need to wait a new cluster creation and setup. If you have set up your tilt cluster and made the current context points to this cluster, you could run: diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index d555a71c..bc554f32 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -4,7 +4,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: - name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} spec: clusterNetwork: pods: @@ -33,7 +34,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KThreesControlPlane metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default + namespace: ${NAMESPACE} spec: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -50,6 +51,7 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate metadata: name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} spec: template: spec: @@ -59,6 +61,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: name: worker-md-0 + namespace: ${NAMESPACE} spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} @@ -91,6 +94,7 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate metadata: name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} spec: template: spec: @@ -100,6 +104,7 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KThreesConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} spec: template: spec: