Support Ingress with SSL passthrough (#271)

- Deploy Ingress resources for NuoDB Admin REST service, NuoDB Admin SQL, and TEs - Automatically inject external-address and external-port TE labels if database.te.ingress.enabled is true - Bump k8s.io/api to v0.19.0 to get support for Ingress v1
nuodb · Feb 14, 2022 · 22984c9 · 22984c9
1 parent 9fce815
commit 22984c9
Show file tree

Hide file tree

Showing 19 changed files with 589 additions and 55 deletions.
diff --git a/go.mod b/go.mod
@@ -8,21 +8,16 @@ require (
 	github.com/aws/aws-sdk-go v1.33.9 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-errors/errors v1.1.1 // indirect
-	github.com/google/go-cmp v0.5.0
-	github.com/googleapis/gnostic v0.4.0 // indirect
+	github.com/google/go-cmp v0.5.5
 	github.com/gruntwork-io/gruntwork-cli v0.6.1 // indirect
 	github.com/gruntwork-io/terratest v0.28.10
 	github.com/imdario/mergo v0.3.10 // indirect
-	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/otiai10/copy v1.2.0
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect
-	golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	gopkg.in/yaml.v2 v2.3.0
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-	k8s.io/api v0.18.6
-	k8s.io/apimachinery v0.18.6
-	k8s.io/client-go v0.18.6 // indirect
-	k8s.io/utils v0.0.0-20200720150651-0bdb4ca86cbc // indirect
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.19.0
+	k8s.io/apimachinery v0.19.0
+	k8s.io/client-go v0.19.0 // indirect
 )
diff --git a/go.sum b/go.sum
diff --git a/scripts/ci/install_deps.sh b/scripts/ci/install_deps.sh
@@ -32,6 +32,12 @@ if [[ "$REQUIRES_MINIKUBE" == "true" ]]; then
 
   sudo chmod 700 $HOME/.kube/config
 
+  # Configure DNS entries for Ingress testing
+  ip=$(sudo minikube ip)
+  echo "$ip api.nuodb.local
+$ip sql.nuodb.local
+$ip demo.nuodb.local" | sudo tee -a /etc/hosts
+
   # Start 'minikube tunnel' so that services with type LoadBalancer are correctly
   # provisioned and routes to the minikube IP are created; 
   # see https://minikube.sigs.k8s.io/docs/handbook/accessing/#using-minikube-tunnel
@@ -55,6 +61,9 @@ if [[ "$REQUIRES_MINIKUBE" == "true" ]]; then
   # get HC Vault for testing
   helm repo add hashicorp https://helm.releases.hashicorp.com
 
+  # get HAProxy for Ingress testing
+  helm repo add haproxytech https://haproxytech.github.io/helm-charts
+
 elif [[ "$REQUIRES_MINISHIFT" == "true" ]]; then
   wget https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz -O /tmp/oc.tar.gz
   tar xzf /tmp/oc.tar.gz -C /tmp --strip-components=1 && chmod +x /tmp/oc && sudo mv /tmp/oc /usr/local/bin

diff --git a/stable/admin/README.md b/stable/admin/README.md
@@ -155,6 +155,13 @@ The following tables list the configurable parameters for the `admin` option of
 | `externalAccess.internalIP` | Whether to use an internal (to the cloud) or external (public) IP address for the load balancer | `nil` |
 | `externalAccess.type` | The service type used to enable external access for NuoDB Admin. The supported types are `NodePort` and `LoadBalancer` (defaults to `LoadBalancer`) | `nil` |
 | `externalAccess.annotations` | Annotations to pass through to the Service of type `LoadBalancer` | `{}` |
+| `ingress.enabled` | Whether to deploy an Ingress resources for the NuoDB Admin. Supported starting with Kubernetes v1.19.0 | `false` |
+| `ingress.api.hostname` | The fully qualified domain name (FQDN) of the network host for the NuoDB Admin REST API | `""` |
+| `ingress.api.className` | The associated IngressClass name defines which Ingress controller will implement the resource | `""` |
+| `ingress.api.annotations` | Custom annotations that are set on the Ingress resource | `{ ingress.kubernetes.io/ssl-passthrough: "true" }` |
+| `ingress.sql.hostname` | The fully qualified domain name (FQDN) of the network host used by SQL clients | `""` |
+| `ingress.sql.className` | The associated IngressClass name defines which Ingress controller will implement the resource | `""` |
+| `ingress.sql.annotations` | Custom annotations that are set on the Ingress resource. SSL passthrough feature should be configured so that SQL clients TLS connections are send directly to the  NuoDB Admin | `{ ingress.kubernetes.io/ssl-passthrough: "true" }` |
 | `resources` | Kubernetes resource requests and limits used for the NuoDB Admin containers | `{}` |
 | `affinity` | Affinity rules for NuoDB Admin | `{}` |
 | `nodeSelector` | Node selector rules for NuoDB Admin | `{}` |

diff --git a/stable/admin/templates/ingress.yaml b/stable/admin/templates/ingress.yaml
@@ -0,0 +1,67 @@
+{{- if .Values.admin.ingress }}
+{{- if eq (include "defaultfalse" .Values.admin.ingress.enabled) "true" }}
+{{- if .Values.admin.ingress.api }}
+{{- if .Values.admin.ingress.api.hostname }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ template "admin.fullname" . }}-api
+  annotations:
+    {{- toYaml .Values.admin.ingress.api.annotations | trim | nindent 4 }}
+  labels:
+    app: {{ template "admin.fullname" . }}
+    chart: {{ template "admin.chart" . }}
+    release: {{ .Release.Name | quote }}
+    domain: {{ .Values.admin.domain }}
+    group: nuodb
+spec:
+  {{- if .Values.admin.ingress.api.className }}
+  ingressClassName: {{ .Values.admin.ingress.api.className | quote }}
+  {{- end }}
+  rules:
+  - host: {{ .Values.admin.ingress.api.hostname | quote }}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: {{ .Values.admin.domain }}-{{ .Values.admin.serviceSuffix.clusterip }}
+            port:
+              name: 8888-tcp
+{{- end }}
+{{- end }}
+{{- if .Values.admin.ingress.sql }}
+{{- if .Values.admin.ingress.sql.hostname }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ template "admin.fullname" . }}
+  annotations:
+    {{- toYaml .Values.admin.ingress.sql.annotations | trim | nindent 4 }}
+  labels:
+    app: {{ template "admin.fullname" . }}
+    chart: {{ template "admin.chart" . }}
+    release: {{ .Release.Name | quote }}
+    domain: {{ .Values.admin.domain }}
+    group: nuodb
+spec:
+  {{- if .Values.admin.ingress.sql.className }}
+  ingressClassName: {{ .Values.admin.ingress.sql.className | quote }}
+  {{- end }}
+  rules:
+  - host: {{ .Values.admin.ingress.sql.hostname | quote }}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: {{ .Values.admin.domain }}-{{ .Values.admin.serviceSuffix.clusterip }}
+            port:
+              name: 48004-tcp
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/stable/admin/values.yaml b/stable/admin/values.yaml
@@ -126,6 +126,36 @@ admin:
   #   type: LoadBalancer
   #   annotations: {}
 
+  # Enable NuoDB external access via Ingress; the provisioned Ingress controller
+  # should support SSL passthrough feature allowing it to send the TLS
+  # connections directly to the NuoDB Admin instead of decrypting the
+  # communication; supported starting with Kubernetes v1.19.0
+  ingress:
+    enabled: false
+    api:
+      # the fully qualified domain name (FQDN) of the network host for the NuoDB
+      # Admin REST API
+      hostname: ""
+      # the associated IngressClass name defines which controller will implement
+      # the resource; for Kubernetes >= 1.18 the ingress class name should be
+      # specified via the className option instead of using annotation
+      className: ""
+      # custom annotations that are set on the Ingress resource
+      annotations: 
+        ingress.kubernetes.io/ssl-passthrough: "true"
+    sql:
+      # the fully qualified domain name (FQDN) of the network host for SQL
+      # clients
+      hostname: ""
+      # For Kubernetes >= 1.18 the ingress class name should be specified via the
+      # ingressClassName option instead of using annotation
+      className: ""
+      # custom annotations that are set on the Ingress resource; SSL passthrough
+      # feature should be configured so that SQL clients TLS connections are
+      # send directly to the  NuoDB Admin
+      annotations: 
+        ingress.kubernetes.io/ssl-passthrough: "true"
+
   persistence:
     size: 1Gi
     accessModes:

diff --git a/stable/database/README.md b/stable/database/README.md
@@ -271,6 +271,10 @@ The following tables list the configurable parameters of the `database` chart an
 | `te.externalAccess.internalIP` | Whether to use an internal (to the cloud) or external (public) IP address for the load balancer. Only applies to external access of type `LoadBalancer` | `nil` |
 | `te.externalAccess.type` | The service type used to enable external database access. The supported types are `NodePort` and `LoadBalancer` (defaults to `LoadBalancer`) | `nil` |
 | `te.externalAccess.annotations` | Annotations to pass through to the Service of type `LoadBalancer` | `{}` |
+| `te.ingress.enabled` | Whether to deploy an Ingress resources for the NuoDB Database. Supported starting with Kubernetes v1.19.0 and NuoDB image version 4.2.3 | `false` |
+| `te.ingress.hostname` | The fully qualified domain name (FQDN) of the network host used by SQL clients to reach this database | `""` |
+| `te.ingress.className` | The associated IngressClass name defines which Ingress controller will implement the resource | `""` |
+| `te.ingress.annotations` | Custom annotations that are set on the Ingress resource. SSL passthrough feature should be configured so that SQL clients TLS connections are send directly to the Transaction Engines | `{ ingress.kubernetes.io/ssl-passthrough: "true" }` |
 | `te.dbServices.enabled` | Whether to deploy clusterip and headless services for direct TE connections (defaults true) | `nil` |
 | `te.logPersistence.enabled` | Whether to enable persistent storage for logs | `false` |
 | `te.logPersistence.overwriteBackoff.copies` | How many copies of the crash directory to keep within windowMinutes | `3` |

diff --git a/stable/database/templates/_helpers.tpl b/stable/database/templates/_helpers.tpl
@@ -425,3 +425,27 @@ networking.gke.io/load-balancer-type: "Internal"
     {{- end }}
   {{- end }}
 {{- end -}}
+
+{{/*
+Renders the Transaction engine labels and injects external address and port
+if Ingress is enabled
+*/}}
+{{- define "database.teLabels" -}}
+  {{- $extraLabels := ""  }}
+  {{- if .Values.database.te.ingress }}
+    {{- if eq (include "defaultfalse" .Values.database.te.ingress.enabled) "true" }}
+      {{- if .Values.database.te.ingress.hostname }}
+        {{- if not (index .Values.database.te.labels "external-address") }}
+          {{- $extraLabels = printf "external-address %s" .Values.database.te.ingress.hostname }}
+        {{- end }}
+        {{- if not (index .Values.database.te.labels "external-port") }}
+          {{- $extraLabels = printf "%s external-port 443" $extraLabels }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+  {{- if or .Values.database.te.labels $extraLabels }}
+- "--labels"
+- "{{ $extraLabels }}{{ include "opt.key-values" .Values.database.te.labels }}"
+  {{- end }}
+{{- end -}}
diff --git a/stable/database/templates/deployment.yaml b/stable/database/templates/deployment.yaml
@@ -86,10 +86,7 @@ spec:
           - "300"
           - "--options"
           - "mem {{ .Values.database.te.resources.requests.memory}} {{- range $opt, $val := .Values.database.te.engineOptions }} {{$opt}} {{$val}} {{- end}}"
-        {{- with .Values.database.te.labels }}
-          - "--labels"
-          - "{{- range $opt, $val := . }} {{$opt}} {{$val}} {{- end}}"
-        {{- end }}
+          {{- include "database.teLabels" . | indent 10 }}
     {{- include "database.otherOptions" .Values.database.te.otherOptions | indent 10 }}
     {{- include "sc.containerSecurityContext" . | indent 8 }}
     {{- include "database.envFrom" . | indent 8 }}

diff --git a/stable/database/templates/ingress.yaml b/stable/database/templates/ingress.yaml
@@ -0,0 +1,34 @@
+{{- if .Values.database.te.ingress }}
+{{- if eq (include "defaultfalse" .Values.database.te.ingress.enabled) "true" }}
+{{- if .Values.database.te.ingress.hostname }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ template "database.fullname" . }}
+  annotations:
+    {{- toYaml .Values.database.te.ingress.annotations | trim | nindent 4 }}
+  labels:
+    app: {{ template "database.fullname" . }}
+    group: nuodb
+    database: {{ .Values.database.name }}
+    domain: {{ .Values.admin.domain }}
+    chart: {{ template "database.chart" . }}
+    release: {{ .Release.Name | quote }}
+spec:
+  {{- if .Values.database.te.ingress.className }}
+  ingressClassName: {{ .Values.database.te.ingress.className | quote }}
+  {{- end }}
+  rules:
+  - host: {{ .Values.database.te.ingress.hostname | quote }}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: {{ template "database.fullname" . }}-{{ default .Values.admin.serviceSuffix.clusterip .Values.database.serviceSuffix.clusterip }}
+            port:
+              name: 48006-tcp
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/stable/database/values.yaml b/stable/database/values.yaml
@@ -442,6 +442,25 @@ database:
       # type: LoadBalancer
       # annotations: {}
 
+    # Enable NuoDB external access via Ingress; the provisioned Ingress
+    # controller should support SSL passthrough feature allowing it to send the
+    # TLS connections directly to the Transaction Engines instead of decrypting
+    # the communication; supported starting with Kubernetes v1.19.0 and NuoDB
+    # image version 4.2.3
+    ingress:
+      enabled: false
+      # the fully qualified domain name (FQDN) of the network host for SQL
+      # clients
+      hostname: ""
+      # For Kubernetes >= 1.18 the ingress class name should be specified via the
+      # ingressClassName option instead of using annotation
+      className: ""
+      # custom annotations that are set on the Ingress resource; SSL passthrough
+      # feature should be configured so that SQL clients TLS connections are
+      # send directly to the Transaction Engines
+      annotations: 
+        ingress.kubernetes.io/ssl-passthrough: "true"
+
     ## By default, database clusterip and headless services for direct TE connections are enabled,
     ## but can be optionally disabled here
     dbServices: {}

diff --git a/test/integration/template_admin_test.go b/test/integration/template_admin_test.go
@@ -859,3 +859,61 @@ func TestAdminServiceAccount(t *testing.T) {
 	})
 
 }
+
+func TestAdminIngressRenders(t *testing.T) {
+	// Path to the helm chart we will test
+	helmChartPath := "../../stable/admin"
+
+	options := &helm.Options{
+		SetValues: map[string]string{
+			"admin.ingress.enabled":       "true",
+			"admin.ingress.sql.hostname":  testlib.ADMIN_SQL_INGRESS_HOSTNAME,
+			"admin.ingress.sql.className": "classSQL",
+		},
+	}
+
+	// verify that Ingress resource for SQL clients is created only
+	output := helm.RenderTemplate(t, options, helmChartPath, "release-name", []string{"templates/ingress.yaml"})
+	for _, obj := range testlib.SplitAndRenderIngress(t, output, 1) {
+		assert.Equal(t, "release-name-nuodb-cluster0-admin", obj.Name)
+		assert.Equal(t, options.SetValues["admin.ingress.sql.className"], *obj.Spec.IngressClassName)
+		assert.Equal(t, options.SetValues["admin.ingress.sql.hostname"], obj.Spec.Rules[0].Host)
+		assert.Equal(t, "nuodb-clusterip", obj.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name)
+		assert.Equal(t, "48004-tcp", obj.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Port.Name)
+		assert.Contains(t, obj.Annotations, "ingress.kubernetes.io/ssl-passthrough")
+	}
+
+	options = &helm.Options{
+		SetValues: map[string]string{
+			"admin.ingress.enabled":             "true",
+			"admin.ingress.api.hostname":        testlib.ADMIN_API_INGRESS_HOSTNAME,
+			"admin.ingress.api.className":       "classAPI",
+			"admin.ingress.api.annotations.bar": "bar",
+			"admin.ingress.sql.hostname":        testlib.ADMIN_SQL_INGRESS_HOSTNAME,
+			"admin.ingress.sql.className":       "classSQL",
+			"admin.ingress.sql.annotations.foo": "foo",
+		},
+	}
+
+	// verify that Ingress resource for the REST service and for SQL clients are created
+	output = helm.RenderTemplate(t, options, helmChartPath, "release-name", []string{"templates/ingress.yaml"})
+	for _, obj := range testlib.SplitAndRenderIngress(t, output, 2) {
+		if strings.HasSuffix(obj.Name, "-api") {
+			assert.Equal(t, options.SetValues["admin.ingress.api.className"], *obj.Spec.IngressClassName)
+			assert.Equal(t, options.SetValues["admin.ingress.api.hostname"], obj.Spec.Rules[0].Host)
+			assert.Equal(t, "nuodb-clusterip", obj.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name)
+			assert.Equal(t, "8888-tcp", obj.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Port.Name)
+			assert.Contains(t, obj.Annotations, "bar")
+			assert.Equal(t, "bar", obj.Annotations["bar"])
+		} else {
+			assert.Equal(t, options.SetValues["admin.ingress.sql.className"], *obj.Spec.IngressClassName)
+			assert.Equal(t, options.SetValues["admin.ingress.sql.hostname"], obj.Spec.Rules[0].Host)
+			assert.Equal(t, "nuodb-clusterip", obj.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name)
+			assert.Equal(t, "48004-tcp", obj.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Port.Name)
+			assert.Contains(t, obj.Annotations, "foo")
+			assert.Equal(t, "foo", obj.Annotations["foo"])
+		}
+		assert.Contains(t, obj.Annotations, "ingress.kubernetes.io/ssl-passthrough")
+	}
+
+}