modules/tls: modularize TLS provisioning (openshift#1811)

* modules/tls: initial commit This adds a new modules under `modules/tls/*` which is necessary for swappable TLS certificates. * modules/bootkube modules/tectonic: configure ingress certificate Currently everything is based off the Kube CA certificate. This changes it by: 1. Configuring a dedicated Dex client CA in Tectonic console pointing to the identity grpc client CA. 2. Configuring a dedicated OIDC CA certificate for the API server. 3. Configuring a dedicated ingress CA certificate for the Tectonic console. This will allow to specify separate CA certificates.
wking · Sep 13, 2017 · db65ea2 · db65ea2
1 parent 12dbe98
commit db65ea2
Show file tree

Hide file tree

Showing 44 changed files with 1,169 additions and 696 deletions.
diff --git a/modules/bootkube/assets.tf b/modules/bootkube/assets.tf
@@ -8,16 +8,16 @@ resource "template_dir" "experimental" {
     etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
     kenc_image          = "${var.container_images["kenc"]}"
 
-    etcd_ca_cert = "${base64encode(data.template_file.etcd_ca_cert_pem.rendered)}"
+    etcd_ca_cert = "${base64encode(var.etcd_ca_cert_pem)}"
 
-    etcd_server_cert = "${base64encode(join("", tls_locally_signed_cert.etcd_server.*.cert_pem))}"
-    etcd_server_key  = "${base64encode(join("", tls_private_key.etcd_server.*.private_key_pem))}"
+    etcd_server_cert = "${base64encode(var.etcd_server_cert_pem)}"
+    etcd_server_key  = "${base64encode(var.etcd_server_key_pem)}"
 
-    etcd_client_cert = "${base64encode(data.template_file.etcd_client_crt.rendered)}"
-    etcd_client_key  = "${base64encode(data.template_file.etcd_client_key.rendered)}"
+    etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"
+    etcd_client_key  = "${base64encode(var.etcd_client_key_pem)}"
 
-    etcd_peer_cert = "${base64encode(join("", tls_locally_signed_cert.etcd_peer.*.cert_pem))}"
-    etcd_peer_key  = "${base64encode(join("", tls_private_key.etcd_peer.*.private_key_pem))}"
+    etcd_peer_cert = "${base64encode(var.etcd_peer_cert_pem)}"
+    etcd_peer_key  = "${base64encode(var.etcd_peer_key_pem)}"
   }
 }
 
@@ -66,7 +66,7 @@ resource "template_dir" "bootkube" {
     etcd_servers = "${
       var.experimental_enabled
         ? format("https://%s:2379", cidrhost(var.service_cidr, 15))
-        : data.template_file.etcd_ca_cert_pem.rendered == ""
+        : var.etcd_ca_cert_pem == ""
           ? join(",", formatlist("http://%s:2379", var.etcd_endpoints))
           : join(",", formatlist("https://%s:2379", var.etcd_endpoints))
       }"
@@ -88,20 +88,21 @@ resource "template_dir" "bootkube" {
     oidc_client_id      = "${var.oidc_client_id}"
     oidc_username_claim = "${var.oidc_username_claim}"
     oidc_groups_claim   = "${var.oidc_groups_claim}"
+    oidc_ca_cert        = "${base64encode(var.oidc_ca_cert)}"
 
-    ca_cert            = "${base64encode(var.ca_cert == "" ? join(" ", tls_self_signed_cert.kube_ca.*.cert_pem) : var.ca_cert)}"
-    apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
-    apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
+    kube_ca_cert       = "${base64encode(var.kube_ca_cert_pem)}"
+    apiserver_key      = "${base64encode(var.apiserver_key_pem)}"
+    apiserver_cert     = "${base64encode(var.apiserver_cert_pem)}"
     serviceaccount_pub = "${base64encode(tls_private_key.service_account.public_key_pem)}"
     serviceaccount_key = "${base64encode(tls_private_key.service_account.private_key_pem)}"
 
-    etcd_ca_flag   = "${data.template_file.etcd_ca_cert_pem.rendered != "" ? "- --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt" : "# no etcd-client-ca.crt given" }"
-    etcd_cert_flag = "${data.template_file.etcd_client_crt.rendered != "" ? "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt" : "# no etcd-client.crt given" }"
-    etcd_key_flag  = "${data.template_file.etcd_client_key.rendered != "" ? "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key" : "# no etcd-client.key given" }"
+    etcd_ca_flag   = "${var.etcd_ca_cert_pem != "" ? "- --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt" : "# no etcd-client-ca.crt given" }"
+    etcd_cert_flag = "${var.etcd_client_cert_pem != "" ? "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt" : "# no etcd-client.crt given" }"
+    etcd_key_flag  = "${var.etcd_client_key_pem != "" ? "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key" : "# no etcd-client.key given" }"
 
-    etcd_ca_cert     = "${base64encode(data.template_file.etcd_ca_cert_pem.rendered)}"
-    etcd_client_cert = "${base64encode(data.template_file.etcd_client_crt.rendered)}"
-    etcd_client_key  = "${base64encode(data.template_file.etcd_client_key.rendered)}"
+    etcd_ca_cert     = "${base64encode(var.etcd_ca_cert_pem)}"
+    etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"
+    etcd_client_key  = "${base64encode(var.etcd_client_key_pem)}"
 
     kubernetes_version = "${replace(var.versions["kubernetes"], "+", "-")}"
 
@@ -123,14 +124,14 @@ resource "template_dir" "bootkube_bootstrap" {
     etcd_servers = "${
       var.experimental_enabled
         ? format("https://%s:2379,https://127.0.0.1:12379", cidrhost(var.service_cidr, 15))
-        : data.template_file.etcd_ca_cert_pem.rendered == ""
+        : var.etcd_ca_cert_pem == ""
           ? join(",", formatlist("http://%s:2379", var.etcd_endpoints))
           : join(",", formatlist("https://%s:2379", var.etcd_endpoints))
       }"
 
-    etcd_ca_flag   = "${data.template_file.etcd_ca_cert_pem.rendered != "" ? "- --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt" : "# no etcd-client-ca.crt given" }"
-    etcd_cert_flag = "${data.template_file.etcd_client_crt.rendered != "" ? "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt" : "# no etcd-client.crt given" }"
-    etcd_key_flag  = "${data.template_file.etcd_client_key.rendered != "" ? "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key" : "# no etcd-client.key given" }"
+    etcd_ca_flag   = "${var.etcd_ca_cert_pem != "" ? "- --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt" : "# no etcd-client-ca.crt given" }"
+    etcd_cert_flag = "${var.etcd_client_cert_pem != "" ? "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt" : "# no etcd-client.crt given" }"
+    etcd_key_flag  = "${var.etcd_client_key_pem != "" ? "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key" : "# no etcd-client.key given" }"
 
     cloud_provider             = "${var.cloud_provider}"
     cloud_provider_config      = "${var.cloud_provider_config}"
@@ -147,9 +148,9 @@ data "template_file" "kubeconfig" {
   template = "${file("${path.module}/resources/kubeconfig")}"
 
   vars {
-    ca_cert      = "${base64encode(var.ca_cert == "" ? join(" ", tls_self_signed_cert.kube_ca.*.cert_pem) : var.ca_cert)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
+    kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}"
+    kubelet_cert = "${base64encode(var.kubelet_cert_pem)}"
+    kubelet_key  = "${base64encode(var.kubelet_key_pem)}"
     server       = "${var.kube_apiserver_url}"
     cluster_name = "${var.cluster_name}"
   }
@@ -178,108 +179,3 @@ resource "local_file" "bootkube_sh" {
 data "template_file" "bootkube_service" {
   template = "${file("${path.module}/resources/bootkube.service")}"
 }
-
-# etcd assets
-data "template_file" "etcd_ca_cert_pem" {
-  template = "${var.experimental_enabled || var.etcd_tls_enabled
-    ? join("", tls_self_signed_cert.etcd_ca.*.cert_pem)
-    : file(var.etcd_ca_cert)
-  }"
-}
-
-data "template_file" "etcd_client_crt" {
-  template = "${var.experimental_enabled || var.etcd_tls_enabled
-    ? join("", tls_locally_signed_cert.etcd_client.*.cert_pem)
-    : file(var.etcd_client_cert)
-  }"
-}
-
-data "template_file" "etcd_client_key" {
-  template = "${var.experimental_enabled || var.etcd_tls_enabled
-    ? join("", tls_private_key.etcd_client.*.private_key_pem)
-    : file(var.etcd_client_key)
-  }"
-}
-
-resource "local_file" "etcd_ca_crt" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled || var.etcd_ca_cert != "/dev/null" ? 1 : 0}"
-  content  = "${data.template_file.etcd_ca_cert_pem.rendered}"
-  filename = "./generated/tls/etcd-client-ca.crt"
-}
-
-resource "local_file" "etcd_client_crt" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled || var.etcd_client_cert != "/dev/null" ? 1 : 0}"
-  content  = "${data.template_file.etcd_client_crt.rendered}"
-  filename = "./generated/tls/etcd-client.crt"
-}
-
-resource "local_file" "etcd_client_key" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled || var.etcd_client_key != "/dev/null" ? 1 : 0}"
-  content  = "${data.template_file.etcd_client_key.rendered}"
-  filename = "./generated/tls/etcd-client.key"
-}
-
-resource "local_file" "etcd_server_crt" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled ? 1 : 0}"
-  content  = "${join("", tls_locally_signed_cert.etcd_server.*.cert_pem)}"
-  filename = "./generated/tls/etcd/server.crt"
-}
-
-resource "local_file" "etcd_server_key" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled ? 1 : 0}"
-  content  = "${join("", tls_private_key.etcd_server.*.private_key_pem)}"
-  filename = "./generated/tls/etcd/server.key"
-}
-
-resource "local_file" "etcd_peer_crt" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled ? 1 : 0}"
-  content  = "${join("", tls_locally_signed_cert.etcd_peer.*.cert_pem)}"
-  filename = "./generated/tls/etcd/peer.crt"
-}
-
-resource "local_file" "etcd_peer_key" {
-  count    = "${var.experimental_enabled || var.etcd_tls_enabled ? 1 : 0}"
-  content  = "${join("", tls_private_key.etcd_peer.*.private_key_pem)}"
-  filename = "./generated/tls/etcd/peer.key"
-}
-
-data "archive_file" "etcd_tls_zip" {
-  type = "zip"
-
-  output_path = "./.terraform/etcd_tls.zip"
-
-  source {
-    filename = "ca.crt"
-    content  = "${data.template_file.etcd_ca_cert_pem.rendered}"
-  }
-
-  source {
-    filename = "server.crt"
-    content  = "${join("", tls_locally_signed_cert.etcd_server.*.cert_pem)}"
-  }
-
-  source {
-    filename = "server.key"
-    content  = "${join("", tls_private_key.etcd_server.*.private_key_pem)}"
-  }
-
-  source {
-    filename = "peer.crt"
-    content  = "${join("", tls_locally_signed_cert.etcd_peer.*.cert_pem)}"
-  }
-
-  source {
-    filename = "peer.key"
-    content  = "${join("", tls_private_key.etcd_peer.*.private_key_pem)}"
-  }
-
-  source {
-    filename = "client.crt"
-    content  = "${data.template_file.etcd_client_crt.rendered}"
-  }
-
-  source {
-    filename = "client.key"
-    content  = "${data.template_file.etcd_client_key.rendered}"
-  }
-}
diff --git a/modules/bootkube/outputs.tf b/modules/bootkube/outputs.tf
@@ -17,77 +17,25 @@
 # interpolated once the assets have all been created.
 output "id" {
   value = "${sha1("
-  ${data.archive_file.etcd_tls_zip.id}
   ${local_file.kubeconfig.id}
   ${local_file.bootkube_sh.id}
   ${template_dir.bootkube.id} ${template_dir.bootkube_bootstrap.id}
   ${join(" ",
-    local_file.etcd_ca_crt.*.id,
-    local_file.etcd_server_crt.*.id,
-    local_file.etcd_server_key.*.id,
-    local_file.etcd_client_crt.*.id,
-    local_file.etcd_client_key.*.id,
-    local_file.etcd_peer_crt.*.id,
-    local_file.etcd_peer_key.*.id,
     template_dir.experimental.*.id,
     template_dir.bootstrap_experimental.*.id,
     template_dir.etcd_experimental.*.id,
     )}
   ")}"
 }
 
-output "etcd_tls_zip" {
-  value = "${data.archive_file.etcd_tls_zip.id != "" ? file("./.terraform/etcd_tls.zip") : ""}"
-}
-
 output "kubeconfig" {
   value = "${data.template_file.kubeconfig.rendered}"
 }
 
-output "ca_cert" {
-  value = "${var.ca_cert == "" ? join(" ", tls_self_signed_cert.kube_ca.*.cert_pem) : var.ca_cert}"
-}
-
-output "ca_key_alg" {
-  value = "${var.ca_cert == "" ? join(" ", tls_self_signed_cert.kube_ca.*.key_algorithm) : var.ca_key_alg}"
-}
-
-output "ca_key" {
-  value = "${var.ca_cert == "" ? join(" ", tls_private_key.kube_ca.*.private_key_pem) : var.ca_key}"
-}
-
 output "systemd_service" {
   value = "${data.template_file.bootkube_service.rendered}"
 }
 
 output "kube_dns_service_ip" {
   value = "${cidrhost(var.service_cidr, 10)}"
 }
-
-output "etcd_ca_crt_pem" {
-  value = "${join("", tls_self_signed_cert.etcd_ca.*.cert_pem)}"
-}
-
-output "etcd_server_crt_pem" {
-  value = "${join("", tls_locally_signed_cert.etcd_server.*.cert_pem)}"
-}
-
-output "etcd_server_key_pem" {
-  value = "${join("", tls_private_key.etcd_server.*.private_key_pem)}"
-}
-
-output "etcd_client_crt_pem" {
-  value = "${join("", tls_locally_signed_cert.etcd_client.*.cert_pem)}"
-}
-
-output "etcd_client_key_pem" {
-  value = "${join("", tls_private_key.etcd_client.*.private_key_pem)}"
-}
-
-output "etcd_peer_crt_pem" {
-  value = "${join("", tls_locally_signed_cert.etcd_peer.*.cert_pem)}"
-}
-
-output "etcd_peer_key_pem" {
-  value = "${join("", tls_private_key.etcd_peer.*.private_key_pem)}"
-}
diff --git a/modules/bootkube/resources/kubeconfig b/modules/bootkube/resources/kubeconfig
@@ -4,7 +4,7 @@ clusters:
 - name: ${cluster_name}
   cluster:
     server: ${server}
-    certificate-authority-data: ${ca_cert}
+    certificate-authority-data: ${kube_ca_cert}
 users:
 - name: kubelet
   user:

diff --git a/modules/bootkube/resources/manifests/kube-apiserver-secret.yaml b/modules/bootkube/resources/manifests/kube-apiserver-secret.yaml
@@ -8,7 +8,8 @@ data:
   apiserver.key: ${apiserver_key}
   apiserver.crt: ${apiserver_cert}
   service-account.pub: ${serviceaccount_pub}
-  ca.crt: ${ca_cert}
+  ca.crt: ${kube_ca_cert}
   etcd-client-ca.crt: ${etcd_ca_cert}
   etcd-client.crt: ${etcd_client_cert}
   etcd-client.key: ${etcd_client_key}
+  oidc-ca.crt: ${oidc_ca_cert}
diff --git a/modules/bootkube/resources/manifests/kube-apiserver.yaml b/modules/bootkube/resources/manifests/kube-apiserver.yaml
@@ -54,7 +54,7 @@ spec:
         - --oidc-client-id=${oidc_client_id}
         - --oidc-username-claim=${oidc_username_claim}
         - --oidc-groups-claim=${oidc_groups_claim}
-        - --oidc-ca-file=/etc/kubernetes/secrets/ca.crt
+        - --oidc-ca-file=/etc/kubernetes/secrets/oidc-ca.crt
         - --cloud-provider=${cloud_provider}
         ${cloud_provider_config_flag}
         - --audit-log-path=/var/log/kubernetes/kube-apiserver-audit.log

diff --git a/modules/bootkube/resources/manifests/kube-controller-manager-secret.yaml b/modules/bootkube/resources/manifests/kube-controller-manager-secret.yaml
@@ -6,4 +6,4 @@ metadata:
 type: Opaque
 data:
   service-account.key: ${serviceaccount_key}
-  ca.crt: ${ca_cert}
+  ca.crt: ${kube_ca_cert}
diff --git a/modules/bootkube/service-account.tf b/modules/bootkube/service-account.tf
@@ -0,0 +1,15 @@
+# Kubernete's Service Account (resources/generated/tls/{service-account.key,service-account.pub})
+resource "tls_private_key" "service_account" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "local_file" "service_account_key" {
+  content  = "${tls_private_key.service_account.private_key_pem}"
+  filename = "./generated/tls/service-account.key"
+}
+
+resource "local_file" "service_account_crt" {
+  content  = "${tls_private_key.service_account.public_key_pem}"
+  filename = "./generated/tls/service-account.pub"
+}