From 3f0b75339e86471df3d845dc8d77e86f107e0422 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 6 May 2024 12:58:20 +0200 Subject: [PATCH 1/3] Add K8s Cron Job to automatically renew certificates --- .../cron_job.tf | 58 +++++++++++++++++++ .../providers.tf | 27 +++++++++ .../variables.tf | 7 +++ .../terragrunt.hcl | 34 +++++++++++ terragrunt/concourse-wg-ci-test/config.yaml | 5 ++ 5 files changed, 131 insertions(+) create mode 100644 terraform-modules/concourse/automatic_certificate_regeneration/cron_job.tf create mode 100644 terraform-modules/concourse/automatic_certificate_regeneration/providers.tf create mode 100644 terraform-modules/concourse/automatic_certificate_regeneration/variables.tf create mode 100644 terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/terragrunt.hcl diff --git a/terraform-modules/concourse/automatic_certificate_regeneration/cron_job.tf b/terraform-modules/concourse/automatic_certificate_regeneration/cron_job.tf new file mode 100644 index 0000000..db8820a --- /dev/null +++ b/terraform-modules/concourse/automatic_certificate_regeneration/cron_job.tf @@ -0,0 +1,58 @@ +resource "kubernetes_cron_job_v1" "automatic_certificate_regeneration" { + metadata { + name = "certificate-regeneration" + namespace = "concourse" + } + spec { + schedule = "@monthly" + failed_jobs_history_limit = 2 + successful_jobs_history_limit = 2 + job_template { + metadata {} + spec { + template { + metadata {} + spec { + restart_policy = "OnFailure" + container { + name = "cert-regen" + image = "yatzek/credhub-cli:2.9.0" + image_pull_policy = "IfNotPresent" + command = ["bash", "-c", "IFS=',' read -r -a CERTIFICATES <<< \"$CERTS_TO_RENEW\"; for cert in \"$${CERTIFICATES[@]}\"; do credhub regenerate -n \"$cert\"; done"] + env { + name = "CERTS_TO_RENEW" + value = var.certificates_to_regenerate + } + env { + name = "CREDHUB_SERVER" + value = "https://credhub.concourse.svc.cluster.local:9000" + } + env { + name = "CREDHUB_CA_CERT" + value_from { + secret_key_ref { + key = "certificate" + name = "credhub-root-ca" + } + } + } + env { + name = "CREDHUB_CLIENT" + value = "credhub_admin_client" + } + env { + name = "CREDHUB_SECRET" + value_from { + secret_key_ref { + key = "password" + name = "credhub-admin-client-credentials" + } + } + } + } + } + } + } + } + } +} diff --git a/terraform-modules/concourse/automatic_certificate_regeneration/providers.tf b/terraform-modules/concourse/automatic_certificate_regeneration/providers.tf new file mode 100644 index 0000000..202c171 --- /dev/null +++ b/terraform-modules/concourse/automatic_certificate_regeneration/providers.tf @@ -0,0 +1,27 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +provider "google" { + project = var.project + region = var.region + zone = var.zone +} + +data "google_client_config" "provider" {} + +data "google_container_cluster" "wg_ci" { + project = var.project + name = var.gke_name + location = var.zone +} + +provider "kubernetes" { + host = "https://${data.google_container_cluster.wg_ci.endpoint}" + token = data.google_client_config.provider.access_token + cluster_ca_certificate = base64decode(data.google_container_cluster.wg_ci.master_auth[0].cluster_ca_certificate) +} diff --git a/terraform-modules/concourse/automatic_certificate_regeneration/variables.tf b/terraform-modules/concourse/automatic_certificate_regeneration/variables.tf new file mode 100644 index 0000000..95c9c13 --- /dev/null +++ b/terraform-modules/concourse/automatic_certificate_regeneration/variables.tf @@ -0,0 +1,7 @@ +variable "project" { nullable = false } +variable "region" { nullable = false } +variable "zone" { nullable = false } + +variable "gke_name" { nullable = false } + +variable "certificates_to_regenerate" { nullable = false } \ No newline at end of file diff --git a/terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/terragrunt.hcl b/terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/terragrunt.hcl new file mode 100644 index 0000000..0e78b3d --- /dev/null +++ b/terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/terragrunt.hcl @@ -0,0 +1,34 @@ +locals { + config = yamldecode(file("../config.yaml")) +} + +remote_state { + backend = "gcs" + generate = { + path = "backend.tf" + if_exists = "overwrite" + } + config = { + bucket = "${local.config.gcs_bucket}" + prefix = "${local.config.gcs_prefix}/automatic-certificate-regeneration" + project = "${local.config.project}" + location = "${local.config.region}" + # use for uniform bucket-level access + # (https://cloud.google.com/storage/docs/uniform-bucket-level-access) + enable_bucket_policy_only = false + } +} + +terraform { + source = local.config.tf_modules.automatic_certificate_regeneration +} + +inputs = { + project = local.config.project + region = local.config.region + zone = local.config.zone + + gke_name = local.config.gke_name + + certificates_to_regenerate = local.config.certificates_to_regenerate +} \ No newline at end of file diff --git a/terragrunt/concourse-wg-ci-test/config.yaml b/terragrunt/concourse-wg-ci-test/config.yaml index 20c7579..96bf647 100644 --- a/terragrunt/concourse-wg-ci-test/config.yaml +++ b/terragrunt/concourse-wg-ci-test/config.yaml @@ -44,6 +44,7 @@ tf_modules: dr_restore: "../../..//terraform-modules/concourse/dr_restore" e2e_test: "../../..//terraform-modules/concourse/e2e_test" secret_rotation_postgresql: "../../..//terraform-modules/concourse/secret_rotation_postgresql" + automatic_certificate_regeneration: "../../..//terraform-modules/concourse/automatic_certificate_regeneration" fly_team: main @@ -122,3 +123,7 @@ wg_ci_cnrm_service_account_permissions: [ "cloudsql.databases.list", "cloudsql.databases.update" ] + +# list of certificates that shall be automatically renewed every month +# enter as one string with a comma-separated list of CredHub certificate names +certificates_to_regenerate: "/concourse/main/test_cert" From b4b3b2295112eeef68cce99b1d2df542f3829d93 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Tue, 7 May 2024 15:35:14 +0200 Subject: [PATCH 2/3] Add docu for certificate regeneration job * and rename Terragrunt file so that the module is not part of the default stack --- docs/concourse/README.md | 3 + docs/concourse/certificate_regeneration.md | 55 +++++++++++++++++++ .../{terragrunt.hcl => cert_regen.hcl} | 0 3 files changed, 58 insertions(+) create mode 100644 docs/concourse/certificate_regeneration.md rename terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/{terragrunt.hcl => cert_regen.hcl} (100%) diff --git a/docs/concourse/README.md b/docs/concourse/README.md index a81fbbc..8011cce 100644 --- a/docs/concourse/README.md +++ b/docs/concourse/README.md @@ -252,3 +252,6 @@ Please see [DR scenario](disaster_recovery.md) for a fully automated recovery pr ## Automated secrets rotation for CloudSQL Please see [Secrets Rotation](secrets_rotation.md) + +## Automated regeneration for certificates stored in CredHub +Please see [Certificate Regeneration](certificate_regeneration.md) \ No newline at end of file diff --git a/docs/concourse/certificate_regeneration.md b/docs/concourse/certificate_regeneration.md new file mode 100644 index 0000000..36c7195 --- /dev/null +++ b/docs/concourse/certificate_regeneration.md @@ -0,0 +1,55 @@ +# Automated certificate regeneration + +You can deploy a K8s CronJob to automatically regenerate certificates which are stored in CredHub. A typical example are load balancer certificates used in a bosh-bootloader environment. The CronJob calls `credhub regenerate `. This will extend the certificate's validity while all other properties remain unchanged. + +The automated regeneration is provided as separate Terragrunt module which must be deployed separately to enable the feature. + +## Prerequisites + +The certificate's CA must be stored in CredHub, and they must be correctly linked. + +## Configuration and deployment + +First, configure the list of certificates in your local `config.yaml`. Define one string with comma-separated certificate names, e.g.: +``` +certificates_to_regenerate: "/concourse/main/cert_1,/concourse/main/cert_2" +``` + +Next, change to the directory `terragrunt//automatic_certificate_regeneration` and call +``` +terragrunt apply --terragrunt-config cert_regen.hcl +``` +You should see that Terraform creates a new resource: +``` +resource "kubernetes_cron_job_v1" "automatic_certificate_regeneration" +(...) +``` +Confirm with `yes`. Afterward, you can see a new CronJob in your K8s deployment: +``` +$ kubectl -n concourse get cronjobs +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +certificate-regeneration @monthly False 0 50m +``` +To test the CronJob, you can invoke it explicitly and check the logs: +``` +kubectl -n concourse create job --from=cronjob/certificate-regeneration cert-regen-job +# wait a few seconds +kubectl -n concourse get pods # search pod "cert-regen-job-" +kubectl -n concourse logs cert-regen-job- +``` +You should see the output from CredHub: +``` +id: 68875a90-c1b7-4391-a2af-bd3a8f33ce47 +name: /concourse/main/cert_1 +type: certificate +value: +version_created_at: "2024-05-07T12:23:43Z" +(...) +``` + +## Deletion + +To delete the CronJob, change to the directory `terragrunt//automatic_certificate_regeneration` and call +``` +terragrunt destroy --terragrunt-config cert_regen.hcl +``` \ No newline at end of file diff --git a/terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/terragrunt.hcl b/terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/cert_regen.hcl similarity index 100% rename from terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/terragrunt.hcl rename to terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/cert_regen.hcl From c157fd7b6aacb9f9d88797a84230e583e5093f23 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Tue, 7 May 2024 16:16:39 +0200 Subject: [PATCH 3/3] Add limitations section --- docs/concourse/certificate_regeneration.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/concourse/certificate_regeneration.md b/docs/concourse/certificate_regeneration.md index 36c7195..a44cd09 100644 --- a/docs/concourse/certificate_regeneration.md +++ b/docs/concourse/certificate_regeneration.md @@ -47,6 +47,16 @@ version_created_at: "2024-05-07T12:23:43Z" (...) ``` +## Limitations + +It's possible to renew CAs with the CronJob. Note however that this would be a one-step renewal process which can result in downtimes. The full 4-step CA renewal process as described on https://github.com/pivotal/credhub-release/blob/main/docs/ca-rotation.md is not implemented. + +If you want to include the CA in the regeneration process, you can add it at the beginning of the list: +``` +certificates_to_regenerate: "/concourse/main/my_CA,/concourse/main/cert_1,/concourse/main/cert_2" +``` +The (self-signed) CA would be regenerated first and then the two certificates would be re-signed with the new CA and the validity would be extended. + ## Deletion To delete the CronJob, change to the directory `terragrunt//automatic_certificate_regeneration` and call