Add agent mTLS (#430)

* Add agent mTLS * review fixes
SeldonIO · Sep 9, 2022 · 4f0b99d · 4f0b99d
1 parent e824324
commit 4f0b99d
Show file tree

Hide file tree

Showing 35 changed files with 2,713 additions and 2,167 deletions.
diff --git a/components/tls/pkg/tls/secret.go b/components/tls/pkg/tls/secret.go
@@ -24,6 +24,9 @@ type TlsSecretHandler struct {
 }
 
 func NewTlsSecretHandler(secretName string, namespace string, clientset kubernetes.Interface, logger log.FieldLogger) (*TlsSecretHandler, error) {
+	if clientset == nil {
+		return nil, fmt.Errorf("Need clientset to run TLS Secrets Handler")
+	}
 	return &TlsSecretHandler{
 		secretName: secretName,
 		clientset:  clientset,

diff --git a/docs/source/contents/getting-started/kubernetes-installation/tls.md b/docs/source/contents/getting-started/kubernetes-installation/tls.md
@@ -26,8 +26,8 @@ Certificates will be loaded and used for the desired gRPC services. The secrets
 
 For the control plane mTLS can be used on gRPC services:
 
- * Scheduler gRPC service: envar service name: SCHEDULER
- * Agent gRPC service: WIP
+ * Scheduler gRPC service: envar service prefix: SCHEDULER
+ * Agent gRPC service: envar service prefix: AGENT
  * Dataflow gRPC service: WIP
 
 ### Helm Control Plane Install
@@ -37,7 +37,9 @@ When installing `seldon-core-v2-setup` you can set the secret names for your cer
 ```bash
 helm install seldon-v2 k8s/helm-charts/seldon-core-v2-setup/ -n seldon-mesh \
      --set scheduler.tls.scheduler.server.secret=seldon-scheduler-server \
-     --set scheduler.tls.scheduler.client.secret=seldon-scheduler-client
+     --set scheduler.tls.scheduler.client.secret=seldon-scheduler-client \
+     --set scheduler.tls.agent.server.secret=seldon-agent-server \
+     --set scheduler.tls.agent.client.secret=seldon-agent-client
 ```
 
 ## Data Plane
@@ -57,13 +59,15 @@ From the project root run:
 
 ### Raw YAML
 
+Raw yaml Certificates can be created with:
+
 ```
-kubectl create -f k8s/yaml/certs.yaml
+kubectl create -f k8s/yaml/certs.yaml -n seldon-mesh
 ```
 
 ### Helm
 
-You can install into the desired namespace, here we use `seldon-mesh` as an example.
+You can install Certificates into the desired namespace, here we use `seldon-mesh` as an example.
 
 ```
 helm install seldon-v2-certs k8s/helm-charts/seldon-core-v2-certs/ -n seldon-mesh

diff --git a/k8s/helm-charts/seldon-core-v2-certs/templates/certs.yaml b/k8s/helm-charts/seldon-core-v2-certs/templates/certs.yaml
@@ -71,3 +71,43 @@ spec:
   issuerRef:
     name: seldon-ca-issuer
     kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: seldon-agent-server
+spec:
+  secretName: seldon-agent-server
+  duration: 2160h
+  renewBefore: 360h
+  commonName: "seldon-mesh.svc"
+  dnsNames:
+  - "seldon-scheduler"    
+  - "seldon-scheduler.svc"
+  - localhost
+  usages:
+  - server auth
+  - client auth
+  issuerRef:
+    name: seldon-ca-issuer
+    kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: seldon-agent-client
+spec:
+  secretName: seldon-agent-client
+  duration: 2160h
+  renewBefore: 360h
+  commonName: "seldon-mesh.svc"
+  dnsNames:
+  - "seldon-scheduler"    
+  - "seldon-scheduler.svc"
+  - localhost
+  usages:
+  - server auth
+  - client auth
+  issuerRef:
+    name: seldon-ca-issuer
+    kind: Issuer
diff --git a/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml b/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml
diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
@@ -45,6 +45,14 @@ rules:
   - secrets
   verbs:
   - get
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
 {{ if .Values.hodometer.enabled }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -505,6 +513,10 @@ spec:
     port: 9005
     protocol: TCP
     targetPort: agent
+  - name: agent-mtls
+    port: 9055
+    protocol: TCP
+    targetPort: agent-mtls
   - name: dataflow
     port: 9008
     protocol: TCP
@@ -884,6 +896,10 @@ spec:
           value: '{{ .Values.scheduler.tls.scheduler.server.secret }}'
         - name: SCHEDULER_TLS_FOLDER_PATH
           value: '{{ .Values.scheduler.tls.scheduler.server.folder }}'
+        - name: AGENT_TLS_SECRET_NAME
+          value: '{{ .Values.scheduler.tls.agent.server.secret }}'
+        - name: AGENT_TLS_FOLDER_PATH
+          value: '{{ .Values.scheduler.tls.agent.server.folder }}'
         - name: ALLOW_PLAINTXT
           value: "true"
         - name: POD_NAMESPACE
@@ -903,6 +919,8 @@ spec:
           name: scheduler-mtls
         - containerPort: 9005
           name: agent
+        - containerPort: 9055
+          name: agent-mtls
         - containerPort: 9008
           name: dataflow
         resources:
@@ -970,6 +988,10 @@ spec:
         value: '{{ .Values.serverConfig.mlserver.serverCapabilities }}'
       - name: SELDON_OVERCOMMIT_PERCENTAGE
         value: '{{ .Values.serverConfig.mlserver.overcommitPercentage }}'
+      - name: AGENT_TLS_SECRET_NAME
+        value: '{{ .Values.scheduler.tls.agent.client.secret }}'
+      - name: AGENT_TLS_FOLDER_PATH
+        value: '{{ .Values.scheduler.tls.agent.client.folder }}'
       - name: SELDON_SERVER_HTTP_PORT
         value: "9000"
       - name: SELDON_SERVER_GRPC_PORT
@@ -982,6 +1004,8 @@ spec:
         value: seldon-scheduler
       - name: SELDON_SCHEDULER_PORT
         value: "9005"
+      - name: SELDON_SCHEDULER_TLS_PORT
+        value: "9055"
       - name: SELDON_METRICS_PORT
         value: "9006"
       - name: SELDON_SERVER_TYPE
@@ -994,6 +1018,10 @@ spec:
         valueFrom:
           fieldRef:
             fieldPath: metadata.name
+      - name: POD_NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
       - name: MEMORY_REQUEST
         valueFrom:
           resourceFieldRef:
@@ -1131,6 +1159,10 @@ spec:
         value: '{{ .Values.serverConfig.triton.serverCapabilities }}'
       - name: SELDON_OVERCOMMIT_PERCENTAGE
         value: '{{ .Values.serverConfig.triton.overcommitPercentage }}'
+      - name: AGENT_TLS_SECRET_NAME
+        value: '{{ .Values.scheduler.tls.agent.client.secret }}'
+      - name: AGENT_TLS_FOLDER_PATH
+        value: '{{ .Values.scheduler.tls.agent.client.folder }}'
       - name: SELDON_SERVER_HTTP_PORT
         value: "9000"
       - name: SELDON_SERVER_GRPC_PORT
@@ -1151,6 +1183,10 @@ spec:
         valueFrom:
           fieldRef:
             fieldPath: metadata.name
+      - name: POD_NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
       - name: MEMORY_REQUEST
         valueFrom:
           resourceFieldRef:

diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml b/k8s/helm-charts/seldon-core-v2-setup/values.yaml
@@ -92,6 +92,13 @@ scheduler:
       client:
         secret:
         folder:
+    agent:
+      server:  
+        secret:
+        folder:
+      client:
+        secret:
+        folder:
 serverConfig:
   rclone:
     image:

diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template b/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template
@@ -92,6 +92,13 @@ scheduler:
       client:
         secret:
         folder:
+    agent:
+      server:  
+        secret:
+        folder:
+      client:
+        secret:
+        folder:
 serverConfig:
   rclone:
     image:

diff --git a/k8s/kustomize/helm-components/patch_mlserver.yaml b/k8s/kustomize/helm-components/patch_mlserver.yaml
@@ -13,6 +13,10 @@ spec:
         value: '{{ .Values.serverConfig.mlserver.serverCapabilities }}'
       - name: SELDON_OVERCOMMIT_PERCENTAGE
         value: '{{ .Values.serverConfig.mlserver.overcommitPercentage }}'
+      - name: AGENT_TLS_SECRET_NAME
+        value: '{{ .Values.scheduler.tls.agent.client.secret }}'
+      - name: AGENT_TLS_FOLDER_PATH
+        value: '{{ .Values.scheduler.tls.agent.client.folder }}'
       image: '{{ .Values.serverConfig.agent.image.registry }}/{{ .Values.serverConfig.agent.image.repository }}:{{ .Values.serverConfig.agent.image.tag }}'
       imagePullPolicy: '{{ .Values.serverConfig.rclone.image.pullPolicy }}'
       name: agent

diff --git a/k8s/kustomize/helm-components/patch_scheduler.yaml b/k8s/kustomize/helm-components/patch_scheduler.yaml
@@ -14,3 +14,7 @@ spec:
             value: '{{ .Values.scheduler.tls.scheduler.server.secret }}'
           - name: SCHEDULER_TLS_FOLDER_PATH
             value: '{{ .Values.scheduler.tls.scheduler.server.folder }}'
+          - name: AGENT_TLS_SECRET_NAME
+            value: '{{ .Values.scheduler.tls.agent.server.secret }}'
+          - name: AGENT_TLS_FOLDER_PATH
+            value: '{{ .Values.scheduler.tls.agent.server.folder }}'
diff --git a/k8s/kustomize/helm-components/patch_triton.yaml b/k8s/kustomize/helm-components/patch_triton.yaml
@@ -13,6 +13,10 @@ spec:
         value: '{{ .Values.serverConfig.triton.serverCapabilities }}'
       - name: SELDON_OVERCOMMIT_PERCENTAGE
         value: '{{ .Values.serverConfig.triton.overcommitPercentage }}'
+      - name: AGENT_TLS_SECRET_NAME
+        value: '{{ .Values.scheduler.tls.agent.client.secret }}'
+      - name: AGENT_TLS_FOLDER_PATH
+        value: '{{ .Values.scheduler.tls.agent.client.folder }}'
       image: '{{ .Values.serverConfig.agent.image.registry }}/{{ .Values.serverConfig.agent.image.repository }}:{{ .Values.serverConfig.agent.image.tag }}'
       imagePullPolicy: '{{ .Values.serverConfig.rclone.image.pullPolicy }}'
       name: agent

diff --git a/k8s/yaml/certs.yaml b/k8s/yaml/certs.yaml
@@ -71,3 +71,43 @@ spec:
   issuerRef:
     name: seldon-ca-issuer
     kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: seldon-agent-server
+spec:
+  secretName: seldon-agent-server
+  duration: 2160h
+  renewBefore: 360h
+  commonName: "seldon-mesh.svc"
+  dnsNames:
+  - "seldon-scheduler"    
+  - "seldon-scheduler.svc"
+  - localhost
+  usages:
+  - server auth
+  - client auth
+  issuerRef:
+    name: seldon-ca-issuer
+    kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: seldon-agent-client
+spec:
+  secretName: seldon-agent-client
+  duration: 2160h
+  renewBefore: 360h
+  commonName: "seldon-mesh.svc"
+  dnsNames:
+  - "seldon-scheduler"    
+  - "seldon-scheduler.svc"
+  - localhost
+  usages:
+  - server auth
+  - client auth
+  issuerRef:
+    name: seldon-ca-issuer
+    kind: Issuer
diff --git a/k8s/yaml/seldon-v2-components.yaml b/k8s/yaml/seldon-v2-components.yaml
@@ -37,6 +37,14 @@ rules:
   - secrets
   verbs:
   - get
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -526,6 +534,10 @@ spec:
     port: 9005
     protocol: TCP
     targetPort: agent
+  - name: agent-mtls
+    port: 9055
+    protocol: TCP
+    targetPort: agent-mtls
   - name: dataflow
     port: 9008
     protocol: TCP
@@ -888,6 +900,10 @@ spec:
           value: ""
         - name: SCHEDULER_TLS_FOLDER_PATH
           value: ""
+        - name: AGENT_TLS_SECRET_NAME
+          value: ""
+        - name: AGENT_TLS_FOLDER_PATH
+          value: ""
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:
@@ -904,6 +920,8 @@ spec:
           name: scheduler-mtls
         - containerPort: 9005
           name: agent
+        - containerPort: 9055
+          name: agent-mtls
         - containerPort: 9008
           name: dataflow
         resources:
@@ -981,8 +999,14 @@ spec:
         value: seldon-scheduler
       - name: SELDON_SCHEDULER_PORT
         value: "9005"
+      - name: SELDON_SCHEDULER_TLS_PORT
+        value: "9055"
       - name: SELDON_METRICS_PORT
         value: "9006"
+      - name: AGENT_TLS_SECRET_NAME
+        value: ""
+      - name: AGENT_TLS_FOLDER_PATH
+        value: ""
       - name: SELDON_SERVER_TYPE
         value: mlserver
       - name: SELDON_ENVOY_HOST
@@ -993,6 +1017,10 @@ spec:
         valueFrom:
           fieldRef:
             fieldPath: metadata.name
+      - name: POD_NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
       - name: MEMORY_REQUEST
         valueFrom:
           resourceFieldRef:
@@ -1134,6 +1162,10 @@ spec:
         value: "9001"
       - name: SELDON_REVERSE_PROXY_GRPC_PORT
         value: "9501"
+      - name: AGENT_TLS_SECRET_NAME
+        value: ""
+      - name: AGENT_TLS_FOLDER_PATH
+        value: ""
       - name: SELDON_SCHEDULER_HOST
         value: seldon-scheduler
       - name: SELDON_SCHEDULER_PORT
@@ -1146,6 +1178,10 @@ spec:
         valueFrom:
           fieldRef:
             fieldPath: metadata.name
+      - name: POD_NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
       - name: MEMORY_REQUEST
         valueFrom:
           resourceFieldRef:
-Original file line number
+Diff line change
@@ Expand Up / @@ -92,6 +92,13 @@ scheduler: @@
           client:
             secret:
             folder:
+        agent:
+          server:
+            secret:
+            folder:
+          client:
+            secret:
+            folder:
     serverConfig:
       rclone:
         image:
@@ Expand Down @@