From f5b57a217b06f1de13c5f2a972de7648cb29e7f8 Mon Sep 17 00:00:00 2001 From: Aditya Konarde Date: Mon, 9 Sep 2019 15:52:54 +0200 Subject: [PATCH 1/2] Add auth proxy for cache, change cache container port --- components/thanos-querier-cache.libsonnet | 2 +- .../thanos-querier-cache-configmap.yaml | 2 +- environments/openshift/kube-thanos.libsonnet | 41 ++++++++++++++- .../manifests/observatorium-template.yaml | 51 +++++++++++++++++-- 4 files changed, 89 insertions(+), 7 deletions(-) diff --git a/components/thanos-querier-cache.libsonnet b/components/thanos-querier-cache.libsonnet index a7c52c5f5..55e834409 100644 --- a/components/thanos-querier-cache.libsonnet +++ b/components/thanos-querier-cache.libsonnet @@ -17,7 +17,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; target: 'query-frontend', http_prefix: null, server: { - http_listen_port: 9091, + http_listen_port: 9090, }, frontend: { split_queries_by_day: true, diff --git a/environments/kubernetes/manifests/thanos-querier-cache-configmap.yaml b/environments/kubernetes/manifests/thanos-querier-cache-configmap.yaml index be71f3602..f53ec5451 100644 --- a/environments/kubernetes/manifests/thanos-querier-cache-configmap.yaml +++ b/environments/kubernetes/manifests/thanos-querier-cache-configmap.yaml @@ -16,7 +16,7 @@ data: "split_queries_by_day": true "http_prefix": null "server": - "http_listen_port": 9091 + "http_listen_port": 9090 "target": "query-frontend" kind: ConfigMap metadata: diff --git a/environments/openshift/kube-thanos.libsonnet b/environments/openshift/kube-thanos.libsonnet index 11b523d0b..16f329ec0 100644 --- a/environments/openshift/kube-thanos.libsonnet +++ b/environments/openshift/kube-thanos.libsonnet @@ -272,6 +272,13 @@ local list = import 'telemeter/lib/list.libsonnet'; roleBinding+: setSubjectNamespace(super.roleBinding) + roleBinding.mixin.metadata.withNamespace(namespace), }, querierCache+: { + // The proxy secret is there to encrypt session created by the oauth proxy. + proxySecret: + secret.new('querier-proxy', { + session_secret: std.base64($.thanos.variables.proxyConfig.sessionSecret), + }) + + secret.mixin.metadata.withNamespace(namespace) + + secret.mixin.metadata.withLabels({ 'app.kubernetes.io/name': 'thanos-querier' }), configmap+: configmap.mixin.metadata.withNamespace(namespace), service+: @@ -294,12 +301,44 @@ local list = import 'telemeter/lib/list.libsonnet'; }, }, }, + ] + [ + container.new('proxy', $.thanos.variables.proxyImage) + + container.withArgs([ + '-provider=openshift', + '-https-address=:%d' % $.thanos.querier.service.spec.ports[2].port, + '-http-address=', + '-email-domain=*', + '-upstream=http://localhost:%d' % $.thanos.querier.service.spec.ports[1].port, + '-openshift-service-account=prometheus-telemeter', + '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}', + '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}', + '-tls-cert=/etc/tls/private/tls.crt', + '-tls-key=/etc/tls/private/tls.key', + '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token', + '-cookie-secret-file=/etc/proxy/secrets/session_secret', + '-openshift-ca=/etc/pki/tls/cert.pem', + '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + '-skip-auth-regex=^/metrics', + ]) + + container.withPorts([ + { name: 'https', containerPort: $.thanos.querier.service.spec.ports[2].port }, + ]) + + container.withVolumeMounts( + [ + volumeMount.new('secret-querier-cache-tls', '/etc/tls/private'), + volumeMount.new('secret-querier-cache-proxy', '/etc/proxy/secrets'), + ] + ), ], }, }, }, } + - deployment.mixin.metadata.withNamespace(namespace), + deployment.mixin.metadata.withNamespace(namespace) + + deployment.mixin.spec.template.spec.withVolumes([ + volume.fromSecret('secret-querier-cache-tls', 'querier-cache-tls'), + volume.fromSecret('secret-querier-cache-proxy', 'querier-cache-proxy'), + ]), }, }, } + { diff --git a/environments/openshift/manifests/observatorium-template.yaml b/environments/openshift/manifests/observatorium-template.yaml index 61e22dac3..a741d6b33 100644 --- a/environments/openshift/manifests/observatorium-template.yaml +++ b/environments/openshift/manifests/observatorium-template.yaml @@ -97,7 +97,7 @@ objects: "split_queries_by_day": true "http_prefix": null "server": - "http_listen_port": 9091 + "http_listen_port": 9090 "target": "query-frontend" kind: ConfigMap metadata: @@ -147,10 +147,53 @@ objects: - mountPath: /etc/cache-config/ name: querier-cache-config readOnly: false + - args: + - -provider=openshift + - -https-address=:9091 + - -http-address= + - -email-domain=* + - -upstream=http://localhost:9090 + - -openshift-service-account=prometheus-telemeter + - '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", + "namespace": "${NAMESPACE}"}' + - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", + "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}' + - -tls-cert=/etc/tls/private/tls.crt + - -tls-key=/etc/tls/private/tls.key + - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - -cookie-secret-file=/etc/proxy/secrets/session_secret + - -openshift-ca=/etc/pki/tls/cert.pem + - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - -skip-auth-regex=^/metrics + image: ${PROXY_IMAGE}:${PROXY_IMAGE_TAG} + name: proxy + ports: + - containerPort: 9091 + name: https + volumeMounts: + - mountPath: /etc/tls/private + name: secret-querier-cache-tls + readOnly: false + - mountPath: /etc/proxy/secrets + name: secret-querier-cache-proxy + readOnly: false volumes: - - configMap: - name: observatorium-cache-conf - name: querier-cache-config + - name: secret-querier-cache-tls + secret: + secretName: querier-cache-tls + - name: secret-querier-cache-proxy + secret: + secretName: querier-cache-proxy +- apiVersion: v1 + data: + session_secret: "" + kind: Secret + metadata: + labels: + app.kubernetes.io/name: thanos-querier + name: querier-proxy + namespace: ${NAMESPACE} + type: Opaque - apiVersion: v1 kind: Service metadata: From 8e67445e9b0f79f1b5212fe8ad363daa1fd4732a Mon Sep 17 00:00:00 2001 From: Aditya Konarde Date: Mon, 9 Sep 2019 18:07:48 +0200 Subject: [PATCH 2/2] Fix serving cert annotation, port name --- components/thanos-querier-cache.libsonnet | 2 +- .../manifests/thanos-querier-cache-service.yaml | 6 +++--- environments/openshift/kube-thanos.libsonnet | 17 +++++++++++++---- .../manifests/observatorium-template.yaml | 11 ++++++++--- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/components/thanos-querier-cache.libsonnet b/components/thanos-querier-cache.libsonnet index 55e834409..ca215d0cf 100644 --- a/components/thanos-querier-cache.libsonnet +++ b/components/thanos-querier-cache.libsonnet @@ -47,7 +47,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; 'observatorium-cache', $.thanos.querierCache.deployment.metadata.labels, [ - ports.newNamed('http', 9091, 9091), + ports.newNamed('cache', 9090, 9090), ], ) + service.mixin.metadata.withNamespace('observatorium') + diff --git a/environments/kubernetes/manifests/thanos-querier-cache-service.yaml b/environments/kubernetes/manifests/thanos-querier-cache-service.yaml index 7555beb0c..186016620 100644 --- a/environments/kubernetes/manifests/thanos-querier-cache-service.yaml +++ b/environments/kubernetes/manifests/thanos-querier-cache-service.yaml @@ -7,8 +7,8 @@ metadata: namespace: observatorium spec: ports: - - name: http - port: 9091 - targetPort: 9091 + - name: cache + port: 9090 + targetPort: 9090 selector: app.kubernetes.io/name: observatorium-querier-cache diff --git a/environments/openshift/kube-thanos.libsonnet b/environments/openshift/kube-thanos.libsonnet index 16f329ec0..f91f30e0a 100644 --- a/environments/openshift/kube-thanos.libsonnet +++ b/environments/openshift/kube-thanos.libsonnet @@ -274,7 +274,7 @@ local list = import 'telemeter/lib/list.libsonnet'; querierCache+: { // The proxy secret is there to encrypt session created by the oauth proxy. proxySecret: - secret.new('querier-proxy', { + secret.new('querier-cache-proxy', { session_secret: std.base64($.thanos.variables.proxyConfig.sessionSecret), }) + secret.mixin.metadata.withNamespace(namespace) + @@ -282,7 +282,16 @@ local list = import 'telemeter/lib/list.libsonnet'; configmap+: configmap.mixin.metadata.withNamespace(namespace), service+: - service.mixin.metadata.withNamespace(namespace), + service.mixin.metadata.withNamespace(namespace) + + service.mixin.metadata.withAnnotations({ + 'service.alpha.openshift.io/serving-cert-secret-name': 'querier-tls', + }) + { + spec+: { + ports+: [ + service.mixin.spec.portsType.newNamed('proxy', 9091, 'https'), + ], + }, + }, deployment+: { spec+: { @@ -305,10 +314,10 @@ local list = import 'telemeter/lib/list.libsonnet'; container.new('proxy', $.thanos.variables.proxyImage) + container.withArgs([ '-provider=openshift', - '-https-address=:%d' % $.thanos.querier.service.spec.ports[2].port, + '-https-address=:%d' % $.thanos.querierCache.service.spec.ports[1].port, '-http-address=', '-email-domain=*', - '-upstream=http://localhost:%d' % $.thanos.querier.service.spec.ports[1].port, + '-upstream=http://localhost:%d' % $.thanos.querierCache.service.spec.ports[0].port, '-openshift-service-account=prometheus-telemeter', '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}', '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}', diff --git a/environments/openshift/manifests/observatorium-template.yaml b/environments/openshift/manifests/observatorium-template.yaml index a741d6b33..f16076876 100644 --- a/environments/openshift/manifests/observatorium-template.yaml +++ b/environments/openshift/manifests/observatorium-template.yaml @@ -191,21 +191,26 @@ objects: metadata: labels: app.kubernetes.io/name: thanos-querier - name: querier-proxy + name: querier-cache-proxy namespace: ${NAMESPACE} type: Opaque - apiVersion: v1 kind: Service metadata: + annotations: + service.alpha.openshift.io/serving-cert-secret-name: querier-tls labels: app.kubernetes.io/name: observatorium-querier-cache name: observatorium-cache namespace: ${NAMESPACE} spec: ports: - - name: http + - name: cache + port: 9090 + targetPort: 9090 + - name: proxy port: 9091 - targetPort: 9091 + targetPort: https selector: app.kubernetes.io/name: observatorium-querier-cache - apiVersion: apps/v1