Enhanced prometheus queries using manual_action (robusta-dev#1156)

* Enhanced prometheus queries using runner_action

Added:
- All prometheus metrics
- Labels and values for specific metric

Dockerfile

@@ -6,26 +6,26 @@ RUN apt-get update \
     && pip3 install --no-cache-dir --upgrade pip \
     && rm -rf /var/lib/apt/lists/*
 
-ENV ENV_TYPE=DEV 
+ENV ENV_TYPE=DEV
 
-RUN mkdir /app 
+RUN mkdir /app
 RUN curl -sSL https://install.python-poetry.org | python3 -
-RUN /root/.local/bin/poetry config virtualenvs.create false 
+RUN /root/.local/bin/poetry config virtualenvs.create false
 WORKDIR /app
 
 # Install gcc to compile rumal.yaml.clib, wheel is missing.
 RUN apt-get update \
     && apt-get install -y --no-install-recommends gcc \
-    && pip3 install --no-cache-dir ruamel.yaml.clib==0.2.6 \
+    && pip3 install --no-cache-dir ruamel.yaml.clib==0.2.8 \
     && apt-get purge -y --auto-remove gcc \
     && rm -rf /var/lib/apt/lists/*
 
 # we install the project requirements and install the app in separate stages to optimize docker layer caching
 COPY pyproject.toml poetry.lock /app/
 RUN /root/.local/bin/poetry install --no-root --no-dev --extras "all"
 COPY src/ /app/src
-RUN /root/.local/bin/poetry install --no-dev --extras "all" 
-    
+RUN /root/.local/bin/poetry install --no-dev --extras "all"
+
 COPY playbooks/ /etc/robusta/playbooks/defaults
 RUN python3 -m pip install --no-cache-dir /etc/robusta/playbooks/defaults
 

helm/robusta/values.yaml

@@ -74,6 +74,8 @@ lightActions:
 - cordon
 - uncordon
 - rollout_restart
+- prometheus_all_available_metrics
+- prometheus_get_series
 
 # install prometheus, alert-manager, and grafana along with Robusta?
 enablePrometheusStack: false

playbooks/robusta_playbooks/prometheus_enrichments.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from datetime import datetime, timedelta
 from typing import List, Optional, Tuple, Union
@@ -6,6 +7,8 @@
 from kubernetes.client.models.v1_service import V1Service
 from prometheus_api_client import PrometheusApiClientException
 from prometrix import PrometheusQueryResult
+from robusta.integrations.prometheus.utils import get_prometheus_connect
+
 from robusta.api import (
     ExecutionBaseEvent,
     MarkdownBlock,
@@ -17,6 +20,8 @@
     action,
     run_prometheus_query,
 )
+from robusta.core.model.base_params import PrometheusParams, ActionParams
+from robusta.core.reporting import JsonBlock
 
 
 def parse_timestamp_string(date_string: str) -> Optional[datetime]:
@@ -42,6 +47,54 @@ def parse_duration(
     return None, None
 
 
+@action
+def prometheus_all_available_metrics(event: ExecutionBaseEvent, prometheus_params: PrometheusParams):
+    result = get_prometheus_all_available_metrics(prometheus_params=prometheus_params)
+
+    event.add_enrichment([JsonBlock(json.dumps({"metrics": result}))])
+
+
+def get_prometheus_all_available_metrics(prometheus_params: PrometheusParams) -> List[str]:
+    try:
+        prom = get_prometheus_connect(prometheus_params=prometheus_params)
+        return prom.all_metrics()
+
+    except Exception as e:
+        logging.error("An error occurred while fetching all available Prometheus metrics", exc_info=True)
+        raise e
+
+
+
+class PrometheusGetSeriesParams(PrometheusParams):
+    """
+    :var match: List of Prometheus series selectors.
+    :var start_time: Optional start time for the query as datetime.
+    :var end_time: Optional end time for the query as datetime.
+    """
+
+    match: List[str]
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+
+
+@action
+def prometheus_get_series(event: ExecutionBaseEvent, prometheus_params: PrometheusGetSeriesParams):
+    result = get_prometheus_series(prometheus_params=prometheus_params)
+
+    event.add_enrichment([JsonBlock(json.dumps({"series": result}))])
+
+
+def get_prometheus_series(prometheus_params: PrometheusGetSeriesParams) -> dict:
+    try:
+        prom = get_prometheus_connect(prometheus_params=prometheus_params)
+        return prom.get_series(match=prometheus_params.match, end_time=prometheus_params.end_time,
+                               start_time=prometheus_params.start_time)
+
+    except Exception as e:
+        logging.error(f"Failed to fetch Prometheus series for match criteria {prometheus_params.match} within the time range {prometheus_params.start_time} - {prometheus_params.end_time}", exc_info=True)
+        raise e
+
+
 @action
 def prometheus_enricher(event: ExecutionBaseEvent, params: PrometheusQueryParams):
     """

pyproject.toml

@@ -65,7 +65,7 @@ cryptography = "^36.0.0"
 PyJWT = "2.4.0"
 fpdf2 = "^2.7.1"
 attrs = "^23.1.0"
-prometrix = "0.1.10"
+prometrix = "0.1.15"
 hikaru-model-26 = "^1.1.1"
 apprise = "^1.5.0"
 rocketchat-api = "^1.30.0"

src/robusta/integrations/prometheus/utils.py

@@ -10,6 +10,7 @@
     PrometheusConfig,
     VictoriaMetricsPrometheusConfig,
 )
+from prometrix.connect.custom_connect import CustomPrometheusConnect
 
 from robusta.core.exceptions import NoPrometheusUrlFound
 from robusta.core.model.base_params import PrometheusParams
@@ -83,7 +84,7 @@ def generate_prometheus_config(prometheus_params: PrometheusParams) -> Prometheu
     return PrometheusConfig(**baseconfig)
 
 
-def get_prometheus_connect(prometheus_params: PrometheusParams) -> "CustomPrometheusConnect":
+def get_prometheus_connect(prometheus_params: PrometheusParams) -> CustomPrometheusConnect:
     # due to cli import dependency errors without prometheus package installed
     from prometrix import get_custom_prometheus_connect
 
@@ -92,7 +93,7 @@ def get_prometheus_connect(prometheus_params: PrometheusParams) -> "CustomPromet
     return get_custom_prometheus_connect(config)
 
 
-def get_prometheus_flags(prom: "CustomPrometheusConnect") -> Optional[Dict]:
+def get_prometheus_flags(prom: CustomPrometheusConnect) -> Optional[Dict]:
     """
     This returns the prometheus flags and stores the prometheus retention time in retentionTime
     """

src/robusta/runner/web.py

@@ -99,6 +99,8 @@ def handle_manual_trigger():
                 action_name=data["action_name"],
                 action_params=data.get("action_params", None),
                 sinks=data.get("sinks", None),
+                sync_response=data.get("sync_response", False),
+                no_sinks=data.get("no_sinks", False) ,
             )
         )