Merge pull request observatorium#58 from aditya-konarde/cache-auth

Add auth proxy for cache, change cache container port
periklis · Sep 10, 2019 · bcd2e67 · bcd2e67
2 parents 30fdec9 + 8e67445
commit bcd2e67
Show file tree

Hide file tree

Showing 5 changed files with 110 additions and 14 deletions.
diff --git a/components/thanos-querier-cache.libsonnet b/components/thanos-querier-cache.libsonnet
@@ -17,7 +17,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
               target: 'query-frontend',
               http_prefix: null,
               server: {
-                http_listen_port: 9091,
+                http_listen_port: 9090,
               },
               frontend: {
                 split_queries_by_day: true,
@@ -47,7 +47,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           'observatorium-cache',
           $.thanos.querierCache.deployment.metadata.labels,
           [
-            ports.newNamed('http', 9091, 9091),
+            ports.newNamed('cache', 9090, 9090),
           ],
         ) +
         service.mixin.metadata.withNamespace('observatorium') +

diff --git a/environments/kubernetes/manifests/thanos-querier-cache-configmap.yaml b/environments/kubernetes/manifests/thanos-querier-cache-configmap.yaml
@@ -16,7 +16,7 @@ data:
       "split_queries_by_day": true
     "http_prefix": null
     "server":
-      "http_listen_port": 9091
+      "http_listen_port": 9090
     "target": "query-frontend"
 kind: ConfigMap
 metadata:

diff --git a/environments/kubernetes/manifests/thanos-querier-cache-service.yaml b/environments/kubernetes/manifests/thanos-querier-cache-service.yaml
@@ -7,8 +7,8 @@ metadata:
   namespace: observatorium
 spec:
   ports:
-  - name: http
-    port: 9091
-    targetPort: 9091
+  - name: cache
+    port: 9090
+    targetPort: 9090
   selector:
     app.kubernetes.io/name: observatorium-querier-cache
diff --git a/environments/openshift/kube-thanos.libsonnet b/environments/openshift/kube-thanos.libsonnet
@@ -272,10 +272,26 @@ local list = import 'telemeter/lib/list.libsonnet';
       roleBinding+: setSubjectNamespace(super.roleBinding) + roleBinding.mixin.metadata.withNamespace(namespace),
     },
     querierCache+: {
+      // The proxy secret is there to encrypt session created by the oauth proxy.
+      proxySecret:
+        secret.new('querier-cache-proxy', {
+          session_secret: std.base64($.thanos.variables.proxyConfig.sessionSecret),
+        }) +
+        secret.mixin.metadata.withNamespace(namespace) +
+        secret.mixin.metadata.withLabels({ 'app.kubernetes.io/name': 'thanos-querier' }),
       configmap+:
         configmap.mixin.metadata.withNamespace(namespace),
       service+:
-        service.mixin.metadata.withNamespace(namespace),
+        service.mixin.metadata.withNamespace(namespace) +
+        service.mixin.metadata.withAnnotations({
+          'service.alpha.openshift.io/serving-cert-secret-name': 'querier-tls',
+        }) + {
+          spec+: {
+            ports+: [
+              service.mixin.spec.portsType.newNamed('proxy', 9091, 'https'),
+            ],
+          },
+        },
       deployment+:
         {
           spec+: {
@@ -294,12 +310,44 @@ local list = import 'telemeter/lib/list.libsonnet';
                       },
                     },
                   },
+                ] + [
+                  container.new('proxy', $.thanos.variables.proxyImage) +
+                  container.withArgs([
+                    '-provider=openshift',
+                    '-https-address=:%d' % $.thanos.querierCache.service.spec.ports[1].port,
+                    '-http-address=',
+                    '-email-domain=*',
+                    '-upstream=http://localhost:%d' % $.thanos.querierCache.service.spec.ports[0].port,
+                    '-openshift-service-account=prometheus-telemeter',
+                    '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}',
+                    '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}',
+                    '-tls-cert=/etc/tls/private/tls.crt',
+                    '-tls-key=/etc/tls/private/tls.key',
+                    '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token',
+                    '-cookie-secret-file=/etc/proxy/secrets/session_secret',
+                    '-openshift-ca=/etc/pki/tls/cert.pem',
+                    '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
+                    '-skip-auth-regex=^/metrics',
+                  ]) +
+                  container.withPorts([
+                    { name: 'https', containerPort: $.thanos.querier.service.spec.ports[2].port },
+                  ]) +
+                  container.withVolumeMounts(
+                    [
+                      volumeMount.new('secret-querier-cache-tls', '/etc/tls/private'),
+                      volumeMount.new('secret-querier-cache-proxy', '/etc/proxy/secrets'),
+                    ]
+                  ),
                 ],
               },
             },
           },
         } +
-        deployment.mixin.metadata.withNamespace(namespace),
+        deployment.mixin.metadata.withNamespace(namespace) +
+        deployment.mixin.spec.template.spec.withVolumes([
+          volume.fromSecret('secret-querier-cache-tls', 'querier-cache-tls'),
+          volume.fromSecret('secret-querier-cache-proxy', 'querier-cache-proxy'),
+        ]),
     },
   },
 } + {

diff --git a/environments/openshift/manifests/observatorium-template.yaml b/environments/openshift/manifests/observatorium-template.yaml
@@ -99,7 +99,7 @@ objects:
         "split_queries_by_day": true
       "http_prefix": null
       "server":
-        "http_listen_port": 9091
+        "http_listen_port": 9090
       "target": "query-frontend"
   kind: ConfigMap
   metadata:
@@ -149,22 +149,70 @@ objects:
           - mountPath: /etc/cache-config/
             name: querier-cache-config
             readOnly: false
+        - args:
+          - -provider=openshift
+          - -https-address=:9091
+          - -http-address=
+          - -email-domain=*
+          - -upstream=http://localhost:9090
+          - -openshift-service-account=prometheus-telemeter
+          - '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}",
+            "namespace": "${NAMESPACE}"}'
+          - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get",
+            "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}'
+          - -tls-cert=/etc/tls/private/tls.crt
+          - -tls-key=/etc/tls/private/tls.key
+          - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
+          - -cookie-secret-file=/etc/proxy/secrets/session_secret
+          - -openshift-ca=/etc/pki/tls/cert.pem
+          - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          - -skip-auth-regex=^/metrics
+          image: ${PROXY_IMAGE}:${PROXY_IMAGE_TAG}
+          name: proxy
+          ports:
+          - containerPort: 9091
+            name: https
+          volumeMounts:
+          - mountPath: /etc/tls/private
+            name: secret-querier-cache-tls
+            readOnly: false
+          - mountPath: /etc/proxy/secrets
+            name: secret-querier-cache-proxy
+            readOnly: false
         volumes:
-        - configMap:
-            name: observatorium-cache-conf
-          name: querier-cache-config
+        - name: secret-querier-cache-tls
+          secret:
+            secretName: querier-cache-tls
+        - name: secret-querier-cache-proxy
+          secret:
+            secretName: querier-cache-proxy
+- apiVersion: v1
+  data:
+    session_secret: ""
+  kind: Secret
+  metadata:
+    labels:
+      app.kubernetes.io/name: thanos-querier
+    name: querier-cache-proxy
+    namespace: ${NAMESPACE}
+  type: Opaque
 - apiVersion: v1
   kind: Service
   metadata:
+    annotations:
+      service.alpha.openshift.io/serving-cert-secret-name: querier-tls
     labels:
       app.kubernetes.io/name: observatorium-querier-cache
     name: observatorium-cache
     namespace: ${NAMESPACE}
   spec:
     ports:
-    - name: http
+    - name: cache
+      port: 9090
+      targetPort: 9090
+    - name: proxy
       port: 9091
-      targetPort: 9091
+      targetPort: https
     selector:
       app.kubernetes.io/name: observatorium-querier-cache
 - apiVersion: apps/v1