From 69066c90ea08fe7907d0082891a3a7f53c67b213 Mon Sep 17 00:00:00 2001
From: Aleksei Kaplin <alexey.kaplin@nutanix.com>
Date: Thu, 14 Oct 2021 05:15:37 +0200
Subject: [PATCH] TMP fix for ml-pipelines

---
 .../kcs/install_kubeflow/monitoring.env.yaml       | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/automation/infrastructure/terraform/kcs/install_kubeflow/monitoring.env.yaml b/automation/infrastructure/terraform/kcs/install_kubeflow/monitoring.env.yaml
index 4f66734..e21d010 100644
--- a/automation/infrastructure/terraform/kcs/install_kubeflow/monitoring.env.yaml
+++ b/automation/infrastructure/terraform/kcs/install_kubeflow/monitoring.env.yaml
@@ -17,6 +17,12 @@ prometheusOperator:
 prometheus:
   prometheusSpec:
     additionalScrapeConfigs:
+      # Custom monitoring: monitoring ml-pipeline (the Kubeflow Pipelines API server)
+      # TODO: remove when fixed: https://github.com/kubeflow/manifests/issues/2011
+      - job_name: 'kubeflow-ml-pipeline'
+        scrape_interval: 15s
+        static_configs:
+          - targets: ['ml-pipeline.kubeflow:8888']
 
       # Scan for all services in "kubeflow..." namespaces with prometheus.io annotations (scrape=true, port, address).
       # Both HTTP and HTTPS
@@ -29,30 +35,38 @@ prometheus:
         kubernetes_sd_configs:
           - role: service
         relabel_configs:
+          # Only "kubeflow..." namespaces
           - source_labels: [__meta_kubernetes_namespace]
             action: keep
             regex: kubeflow(.*)
+          # Only with prometheus.io/scrape = true
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
             action: keep
             regex: true
+          # Use scheme
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
             action: replace
             target_label: __scheme__
             regex: (https?)
+          # Use path (/metrics)
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
             action: replace
             target_label: __metrics_path__
             regex: (.+)
+          # Use address & port
           - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
             action: replace
             target_label: __address__
             regex: ([^:]+)(?::\d+)?;(\d+)
             replacement: $1:$2
+          # Remove prefix from labels
           - action: labelmap
             regex: __meta_kubernetes_service_label_(.+)
+          # Save namespace label
           - source_labels: [__meta_kubernetes_namespace]
             action: replace
             target_label: kubernetes_namespace
+          # Save service name label
           - source_labels: [__meta_kubernetes_service_name]
             action: replace
             target_label: service_name