From 8bc0bb951c49ea9dc5277917005e2dde6afc2912 Mon Sep 17 00:00:00 2001 From: Arik Alon Date: Thu, 14 Dec 2023 18:27:59 +0200 Subject: [PATCH 1/3] change demo-alert job default user --- src/robusta/cli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/robusta/cli/main.py b/src/robusta/cli/main.py index 74bf75a3a..2279281a3 100755 --- a/src/robusta/cli/main.py +++ b/src/robusta/cli/main.py @@ -11,7 +11,7 @@ import certifi import typer import yaml -from hikaru.model.rel_1_26 import Container, Job, JobSpec, ObjectMeta, PodSpec, PodTemplateSpec +from hikaru.model.rel_1_26 import Container, Job, JobSpec, ObjectMeta, PodSpec, PodTemplateSpec, SecurityContext from kubernetes import client, config from pydantic import BaseModel, Extra @@ -501,6 +501,7 @@ def demo_alert( name="alert-curl", image=image, command=command, + securityContext=SecurityContext(runAsUser=2000), ) ], restartPolicy="Never", From 05ae3ca1a8496d2b9d9cfea8e0ea4a34c5d36f27 Mon Sep 17 00:00:00 2001 From: Arik Alon Date: Sun, 17 Dec 2023 13:58:26 +0200 Subject: [PATCH 2/3] multi resources trigger events caching --- .../triggers/kubernetes.rst | 51 +++++++++++++++++ helm/robusta/values.yaml | 14 ++--- .../core/discovery/top_service_resolver.py | 7 +++ src/robusta/core/model/env_vars.py | 1 - src/robusta/core/playbooks/base_trigger.py | 26 ++------- .../playbooks/internal/discovery_events.py | 17 +++--- .../playbooks/playbooks_event_handler_impl.py | 14 +++-- src/robusta/core/triggers/custom_triggers.py | 2 + .../core/triggers/error_event_trigger.py | 14 ++--- .../core/triggers/helm_releases_triggers.py | 10 ++-- .../core/triggers/job_failed_trigger.py | 8 ++- .../core/triggers/multi_resoources_trigger.py | 55 +++++++++++++++++++ .../core/triggers/oom_killed_trigger_base.py | 8 +-- .../core/triggers/pod_crash_loop_trigger.py | 8 ++- .../core/triggers/pod_image_pull_backoff.py | 7 ++- .../integrations/kubernetes/base_triggers.py | 18 ++++-- .../integrations/prometheus/trigger.py | 15 +++-- src/robusta/runner/config_loader.py | 8 ++- 18 files changed, 203 insertions(+), 80 deletions(-) create mode 100644 src/robusta/core/triggers/multi_resoources_trigger.py diff --git a/docs/playbook-reference/triggers/kubernetes.rst b/docs/playbook-reference/triggers/kubernetes.rst index 606e9ff83..6fe72d4af 100644 --- a/docs/playbook-reference/triggers/kubernetes.rst +++ b/docs/playbook-reference/triggers/kubernetes.rst @@ -177,6 +177,57 @@ Low-level Triggers Low-level triggers fire on the raw creation, deletion, and modification of resources in your cluster. They can be noisy compared to other triggers, as they fire on even the smallest change to a resource. +Multi-Resource Triggers +------------------------- + +.. _on_kubernetes_resource_operation: + +.. details:: on_kubernetes_resource_operation + + ``on_kubernetes_resource_operation`` fires when one of the specified resources, had one of the specified operations. + + * ``operations``: List of `operations`. If empty, all `operations` are included. Options: + * ``create`` + * ``update`` + * ``delete`` + * ``resources``: List of Kubernetes `resources`. If empty, all `resources` are included. Options: + * ``deployment`` + * ``pod`` + * ``job`` + * ``node`` + * ``replicaset`` + * ``statefulset`` + * ``daemonset`` + * ``ingress`` + * ``service`` + * ``event`` + * ``horizontalpodautoscaler`` + * ``clusterrole`` + * ``clusterrolebinding`` + * ``namespace`` + * ``serviceaccount`` + * ``persistentvolume`` + * ``configmap`` + + Example playbook: + + .. code-block:: yaml + + customPlaybooks: + - triggers: + - on_kubernetes_resource_operation: + resources: ["deployment"] + operations: ["update"] + actions: + - create_finding: + title: "Deployment $name on namespace $namespace updated" + aggregation_key: "Deployment Update" + + + +Single Resource Triggers +------------------------- + .. jinja:: :inline-ctx: { "resource_name" : "Pod", "related_actions" : ["Pod Enrichers (General)", "pod_events_enricher"] } :header_update_levels: diff --git a/helm/robusta/values.yaml b/helm/robusta/values.yaml index ddda59a2a..23d820974 100644 --- a/helm/robusta/values.yaml +++ b/helm/robusta/values.yaml @@ -383,9 +383,8 @@ platformPlaybooks: sinks: - "robusta_ui_sink" - triggers: - - on_deployment_all_changes: {} - - on_daemonset_all_changes: {} - - on_statefulset_all_changes: {} + - on_kubernetes_resource_operation: + resources: ["deployment", "daemonset", "statefulset"] actions: - resource_babysitter: {} sinks: @@ -400,13 +399,8 @@ platformPlaybooks: sinks: - "robusta_ui_sink" - triggers: - - on_deployment_all_changes: {} - - on_daemonset_all_changes: {} - - on_statefulset_all_changes: {} - - on_replicaset_all_changes: {} - - on_pod_all_changes: {} - - on_node_all_changes: {} - - on_job_all_changes: {} + - on_kubernetes_resource_operation: + resources: ["deployment", "replicaset", "daemonset", "statefulset", "pod", "node", "job" ] actions: - resource_events_diff: {} - triggers: diff --git a/src/robusta/core/discovery/top_service_resolver.py b/src/robusta/core/discovery/top_service_resolver.py index f073958f4..dd6b9e4bc 100644 --- a/src/robusta/core/discovery/top_service_resolver.py +++ b/src/robusta/core/discovery/top_service_resolver.py @@ -74,3 +74,10 @@ def add_cached_resource(cls, resource: TopLevelResource): cls.__recent_resource_updates[resource.get_resource_key()] = CachedResourceInfo( resource=resource, event_time=time.time() ) + + @classmethod + def remove_cached_resource(cls, resource: TopLevelResource): + if resource in cls.__namespace_to_resource[resource.namespace]: + cls.__namespace_to_resource[resource.namespace].remove(resource) + with cls.__cached_updates_lock: + cls.__recent_resource_updates.pop(resource.get_resource_key(), None) diff --git a/src/robusta/core/model/env_vars.py b/src/robusta/core/model/env_vars.py index 27d3eef8e..726ee3396 100644 --- a/src/robusta/core/model/env_vars.py +++ b/src/robusta/core/model/env_vars.py @@ -34,7 +34,6 @@ def load_bool(env_var, default: bool): DEFAULT_TIMEZONE = pytz.timezone(os.environ.get("DEFAULT_TIMEZONE", "UTC")) NUM_EVENT_THREADS = int(os.environ.get("NUM_EVENT_THREADS", 20)) INCOMING_EVENTS_QUEUE_MAX_SIZE = int(os.environ.get("INCOMING_EVENTS_QUEUE_MAX_SIZE", 500)) -EVENT_PARSING_WORKERS = int(os.environ.get("EVENT_PARSING_WORKERS", 1)) FLOAT_PRECISION_LIMIT = int(os.environ.get("FLOAT_PRECISION_LIMIT", 11)) diff --git a/src/robusta/core/playbooks/base_trigger.py b/src/robusta/core/playbooks/base_trigger.py index e2c8a2cd8..0c1b498d3 100644 --- a/src/robusta/core/playbooks/base_trigger.py +++ b/src/robusta/core/playbooks/base_trigger.py @@ -1,13 +1,10 @@ import abc -from concurrent.futures.process import ProcessPoolExecutor -from typing import Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type from pydantic import BaseModel -from robusta.core.model.env_vars import EVENT_PARSING_WORKERS from robusta.core.model.events import ExecutionBaseEvent from robusta.core.reporting.base import Finding -from robusta.patch.patch import create_monkey_patches from robusta.utils.documented_pydantic import DocumentedModel @@ -22,32 +19,17 @@ def get_event_description(self) -> str: return "NA" -build_execution_event_process_pool = ProcessPoolExecutor(max_workers=EVENT_PARSING_WORKERS) - - class BaseTrigger(DocumentedModel): def get_trigger_event(self) -> str: pass - def should_fire(self, event: TriggerEvent, playbook_id: str): + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): return True def build_execution_event( - self, event: TriggerEvent, sink_findings: Dict[str, List[Finding]] - ) -> Optional[ExecutionBaseEvent]: - return build_execution_event_process_pool.submit( - self._build_execution_event_trampoline, event, sink_findings - ).result() - - def _build_execution_event_trampoline(self, event: TriggerEvent, sink_findings: Dict[str, List[Finding]]): - create_monkey_patches() - return self._build_execution_event(event, sink_findings) - - def _build_execution_event( - self, event: TriggerEvent, sink_findings: Dict[str, List[Finding]] + self, event: TriggerEvent, sink_findings: Dict[str, List[Finding]], build_context: Dict[str, Any] ) -> Optional[ExecutionBaseEvent]: - # This is meant for running in a separate process - raise NotImplementedError + pass @staticmethod def get_execution_event_type() -> Type[ExecutionBaseEvent]: diff --git a/src/robusta/core/playbooks/internal/discovery_events.py b/src/robusta/core/playbooks/internal/discovery_events.py index 0526facbd..9b0ba6066 100644 --- a/src/robusta/core/playbooks/internal/discovery_events.py +++ b/src/robusta/core/playbooks/internal/discovery_events.py @@ -20,17 +20,18 @@ @action def cluster_discovery_updates(event: KubernetesAnyChangeEvent): if ( - event.operation in [K8sOperationType.CREATE, K8sOperationType.UPDATE] - and event.obj.kind in ["Deployment", "ReplicaSet", "DaemonSet", "StatefulSet", "Pod", "Job"] + event.obj.kind in ["Deployment", "ReplicaSet", "DaemonSet", "StatefulSet", "Pod", "Job"] and not event.obj.metadata.ownerReferences ): - TopServiceResolver.add_cached_resource( - TopLevelResource( - name=event.obj.metadata.name, - resource_type=event.obj.kind, - namespace=event.obj.metadata.namespace, - ) + resource = TopLevelResource( + name=event.obj.metadata.name, + resource_type=event.obj.kind, + namespace=event.obj.metadata.namespace, ) + if event.operation in [K8sOperationType.CREATE, K8sOperationType.UPDATE]: + TopServiceResolver.add_cached_resource(resource) + elif event.operation == K8sOperationType.DELETE: + TopServiceResolver.remove_cached_resource(resource) @action diff --git a/src/robusta/core/playbooks/playbooks_event_handler_impl.py b/src/robusta/core/playbooks/playbooks_event_handler_impl.py index 1abf0a188..2bc7c74f7 100644 --- a/src/robusta/core/playbooks/playbooks_event_handler_impl.py +++ b/src/robusta/core/playbooks/playbooks_event_handler_impl.py @@ -17,7 +17,6 @@ from robusta.core.reporting import MarkdownBlock from robusta.core.reporting.base import Finding from robusta.core.reporting.consts import SYNC_RESPONSE_SINK -from robusta.core.sinks.robusta import RobustaSink from robusta.core.sinks.robusta.dal.model_conversion import ModelConversion from robusta.model.alert_relabel_config import AlertRelabel from robusta.model.config import Registry @@ -46,12 +45,13 @@ def handle_trigger(self, trigger_event: TriggerEvent) -> Optional[Dict[str, Any] execution_response = None execution_event: Optional[ExecutionBaseEvent] = None sink_findings: Dict[str, List[Finding]] = defaultdict(list) + build_context: Dict[str, Any] = {} for playbook in playbooks: - fired_trigger = self.__get_fired_trigger(trigger_event, playbook.triggers, playbook.get_id()) + fired_trigger = self.__get_fired_trigger(trigger_event, playbook.triggers, playbook.get_id(), build_context) if fired_trigger: execution_event = None try: - execution_event = fired_trigger.build_execution_event(trigger_event, sink_findings) + execution_event = fired_trigger.build_execution_event(trigger_event, sink_findings, build_context) # sink_findings needs to be shared between playbooks. # build_execution_event returns a different instance because it's running in a child process execution_event.sink_findings = sink_findings @@ -281,9 +281,10 @@ def __get_fired_trigger( trigger_event: TriggerEvent, playbook_triggers: List[Trigger], playbook_id: str, + build_context: Dict[str, Any], ) -> Optional[BaseTrigger]: for trigger in playbook_triggers: - if trigger.get().should_fire(trigger_event, playbook_id): + if trigger.get().should_fire(trigger_event, playbook_id, build_context): return trigger.get() return None @@ -307,6 +308,11 @@ def __handle_findings(self, execution_event: ExecutionBaseEvent): # Each sink has a different findings, but enrichments are shared finding_copy = copy.deepcopy(finding) sink.write_finding(finding_copy, self.registry.get_sinks().platform_enabled) + + sink_info = sinks_info[sink_name] + sink_info.type = sink.__class__.__name__ + sink_info.findings_count += 1 + if sink.params.stop: return diff --git a/src/robusta/core/triggers/custom_triggers.py b/src/robusta/core/triggers/custom_triggers.py index 8c1cf9714..2425ac178 100644 --- a/src/robusta/core/triggers/custom_triggers.py +++ b/src/robusta/core/triggers/custom_triggers.py @@ -10,6 +10,7 @@ WarningEventUpdateTrigger, ) from robusta.core.triggers.job_failed_trigger import JobFailedTrigger +from robusta.core.triggers.multi_resoources_trigger import MultiResourceTrigger from robusta.core.triggers.pod_crash_loop_trigger import PodCrashLoopTrigger from robusta.core.triggers.pod_image_pull_backoff import PodImagePullBackoffTrigger from robusta.core.triggers.pod_oom_killed_trigger import PodOOMKilledTrigger @@ -25,3 +26,4 @@ class CustomTriggers(BaseModel): on_job_failure: Optional[JobFailedTrigger] on_pod_oom_killed: Optional[PodOOMKilledTrigger] on_container_oom_killed: Optional[ContainerOOMKilledTrigger] + on_kubernetes_resource_operation: Optional[MultiResourceTrigger] diff --git a/src/robusta/core/triggers/error_event_trigger.py b/src/robusta/core/triggers/error_event_trigger.py index 35509bfc3..095251dce 100644 --- a/src/robusta/core/triggers/error_event_trigger.py +++ b/src/robusta/core/triggers/error_event_trigger.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Any, Dict, List from robusta.core.discovery.top_service_resolver import TopServiceResolver from robusta.core.playbooks.base_trigger import TriggerEvent @@ -33,15 +33,18 @@ def __init__( self.exclude = exclude self.include = include - def should_fire(self, event: TriggerEvent, playbook_id: str): - should_fire = super().should_fire(event, playbook_id) + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): + should_fire = super().should_fire(event, playbook_id, build_context) if not should_fire: return should_fire if not isinstance(event, K8sTriggerEvent): return False - exec_event = self.build_execution_event(event, {}) + if event.k8s_payload.obj.get("type", None) != "Warning": + return False + + exec_event = self.build_execution_event(event, {}, build_context) if not isinstance(exec_event, EventChangeEvent): return False @@ -49,9 +52,6 @@ def should_fire(self, event: TriggerEvent, playbook_id: str): if not exec_event.obj or not exec_event.obj.regarding: return False - if exec_event.get_event().type != "Warning": - return False - if self.operations and exec_event.operation.value not in self.operations: return False diff --git a/src/robusta/core/triggers/helm_releases_triggers.py b/src/robusta/core/triggers/helm_releases_triggers.py index 9738a2ba6..634c42880 100644 --- a/src/robusta/core/triggers/helm_releases_triggers.py +++ b/src/robusta/core/triggers/helm_releases_triggers.py @@ -1,7 +1,7 @@ -import logging import sys -from typing import Optional, List, Dict from datetime import datetime +from typing import Any, Dict, List, Optional + import pytz from attr import dataclass from pydantic import BaseModel @@ -68,7 +68,7 @@ class HelmReleaseBaseTrigger(BaseTrigger): def get_trigger_event(self): return HelmReleasesTriggerEvent.__name__ - def should_fire(self, event: TriggerEvent, playbook_id: str): + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): if not isinstance(event, HelmReleasesTriggerEvent): return False @@ -116,8 +116,8 @@ def should_fire(self, event: TriggerEvent, playbook_id: str): return can_fire - def _build_execution_event( - self, event: HelmReleasesTriggerEvent, sink_findings: Dict[str, List[Finding]] + def build_execution_event( + self, event: HelmReleasesTriggerEvent, sink_findings: Dict[str, List[Finding]], build_context: Dict[str, Any] ) -> Optional[ExecutionBaseEvent]: if not isinstance(event, HelmReleasesTriggerEvent): return diff --git a/src/robusta/core/triggers/job_failed_trigger.py b/src/robusta/core/triggers/job_failed_trigger.py index 89f74feb8..c6ad452ef 100644 --- a/src/robusta/core/triggers/job_failed_trigger.py +++ b/src/robusta/core/triggers/job_failed_trigger.py @@ -1,3 +1,5 @@ +from typing import Any, Dict + from hikaru.model.rel_1_26 import Job from robusta.core.playbooks.base_trigger import TriggerEvent @@ -22,15 +24,15 @@ def __init__( labels_selector=labels_selector, ) - def should_fire(self, event: TriggerEvent, playbook_id: str): - should_fire = super().should_fire(event, playbook_id) + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): + should_fire = super().should_fire(event, playbook_id, build_context) if not should_fire: return should_fire if not isinstance(event, K8sTriggerEvent): return False - exec_event = self.build_execution_event(event, {}) + exec_event = self.build_execution_event(event, {}, build_context) if not isinstance(exec_event, JobChangeEvent): return False diff --git a/src/robusta/core/triggers/multi_resoources_trigger.py b/src/robusta/core/triggers/multi_resoources_trigger.py new file mode 100644 index 000000000..a2b1045d3 --- /dev/null +++ b/src/robusta/core/triggers/multi_resoources_trigger.py @@ -0,0 +1,55 @@ +from typing import Any, Dict, List + +from robusta.core.playbooks.base_trigger import TriggerEvent +from robusta.integrations.kubernetes.autogenerated.triggers import KubernetesAnyAllChangesTrigger +from robusta.integrations.kubernetes.base_triggers import K8sTriggerEvent + + +class MultiResourceTrigger(KubernetesAnyAllChangesTrigger): + """ + Can be configured to fire on a set of Kubernetes resources, for all or a subset of operations + + :var resources: List of Kubernetes resources. If empty, all resources are included. + Allowed values: deployment, pod, job, node, replicaset, statefulset, daemonset, ingress, service, + event, horizontalpodautoscaler, clusterrole, clusterrolebinding, namespace, + serviceaccount, persistentvolume, configmap + :var operations: List of operations. If empty, all operations are included. Allowed values: create, update, delete + + :example resources: ["deployment", "job", "ingress"] + :example operations: ["create", "update"] + """ + + resources: List[str] = (None,) + operations: List[str] = None + + def __init__( + self, + name_prefix: str = None, + namespace_prefix: str = None, + labels_selector: str = None, + resources: List[str] = None, + operations: List[str] = None, + ): + super().__init__( + name_prefix=name_prefix, + namespace_prefix=namespace_prefix, + labels_selector=labels_selector, + ) + self.resources = [resource.lower() for resource in resources] if resources else [] + self.operations = [op.lower() for op in operations] if operations else [] + + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): + should_fire = super().should_fire(event, playbook_id, build_context) + if not should_fire: + return should_fire + + if not isinstance(event, K8sTriggerEvent): + return False + + if self.resources and event.k8s_payload.kind.lower() not in self.resources: + return False + + if self.operations and event.k8s_payload.operation.lower() not in self.operations: + return False + + return True diff --git a/src/robusta/core/triggers/oom_killed_trigger_base.py b/src/robusta/core/triggers/oom_killed_trigger_base.py index af0fee3e3..991c89087 100644 --- a/src/robusta/core/triggers/oom_killed_trigger_base.py +++ b/src/robusta/core/triggers/oom_killed_trigger_base.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Dict, List +from typing import Any, Dict, List from hikaru.model.rel_1_26 import ContainerStatus, Pod from pydantic import BaseModel @@ -41,15 +41,15 @@ def __init__( # pydantic not automatically converting exclude, exclude here is just a list of dicts according to runtime self.exclude = [Exclude(**excluded) for excluded in exclude] if exclude else [] - def should_fire(self, event: TriggerEvent, playbook_id: str): - should_fire = super().should_fire(event, playbook_id) + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): + should_fire = super().should_fire(event, playbook_id, build_context) if not should_fire: return should_fire if not isinstance(event, K8sTriggerEvent): return False - exec_event = self.build_execution_event(event, {}) + exec_event = self.build_execution_event(event, {}, build_context) if not isinstance(exec_event, PodChangeEvent): return False diff --git a/src/robusta/core/triggers/pod_crash_loop_trigger.py b/src/robusta/core/triggers/pod_crash_loop_trigger.py index 38ca0e24f..9135fba03 100644 --- a/src/robusta/core/triggers/pod_crash_loop_trigger.py +++ b/src/robusta/core/triggers/pod_crash_loop_trigger.py @@ -1,3 +1,5 @@ +from typing import Any, Dict + from robusta.core.playbooks.base_trigger import TriggerEvent from robusta.integrations.kubernetes.autogenerated.triggers import PodChangeEvent, PodUpdateTrigger from robusta.integrations.kubernetes.base_triggers import K8sTriggerEvent @@ -33,15 +35,15 @@ def __init__( self.restart_reason = restart_reason self.restart_count = restart_count - def should_fire(self, event: TriggerEvent, playbook_id: str): - should_fire = super().should_fire(event, playbook_id) + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): + should_fire = super().should_fire(event, playbook_id, build_context) if not should_fire: return should_fire if not isinstance(event, K8sTriggerEvent): return False - exec_event = self.build_execution_event(event, {}) + exec_event = self.build_execution_event(event, {}, build_context) if not isinstance(exec_event, PodChangeEvent): return False diff --git a/src/robusta/core/triggers/pod_image_pull_backoff.py b/src/robusta/core/triggers/pod_image_pull_backoff.py index 60a8bdef4..9ab3d3fd6 100644 --- a/src/robusta/core/triggers/pod_image_pull_backoff.py +++ b/src/robusta/core/triggers/pod_image_pull_backoff.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Any, Dict from robusta.core.playbooks.base_trigger import TriggerEvent from robusta.integrations.kubernetes.api_client_utils import parse_kubernetes_datetime_to_ms @@ -32,15 +33,15 @@ def __init__( self.rate_limit = rate_limit self.fire_delay = fire_delay - def should_fire(self, event: TriggerEvent, playbook_id: str): - should_fire = super().should_fire(event, playbook_id) + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): + should_fire = super().should_fire(event, playbook_id, build_context) if not should_fire: return should_fire if not isinstance(event, K8sTriggerEvent): return False - exec_event = self.build_execution_event(event, {}) + exec_event = self.build_execution_event(event, {}, build_context) if not isinstance(exec_event, PodChangeEvent): return False diff --git a/src/robusta/integrations/kubernetes/base_triggers.py b/src/robusta/integrations/kubernetes/base_triggers.py index fdb3ce196..5351984a1 100644 --- a/src/robusta/integrations/kubernetes/base_triggers.py +++ b/src/robusta/integrations/kubernetes/base_triggers.py @@ -14,6 +14,9 @@ from robusta.integrations.kubernetes.autogenerated.models import get_api_version from robusta.integrations.kubernetes.model_not_found_exception import ModelNotFoundException +OBJ = "obj" +OLD_OBJ = "old_obj" + class IncomingK8sEventPayload(BaseModel): """ @@ -64,7 +67,7 @@ def __init__(self, *args, **data): def get_trigger_event(self): return K8sTriggerEvent.__name__ - def should_fire(self, event: TriggerEvent, playbook_id: str): + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): if not isinstance(event, K8sTriggerEvent): return False @@ -117,8 +120,8 @@ def __parse_kubernetes_objs(cls, k8s_payload: IncomingK8sEventPayload): old_obj = cls.__load_hikaru_obj(k8s_payload.oldObj, model_class) return obj, old_obj - def _build_execution_event( - self, event: K8sTriggerEvent, sink_findings: Dict[str, List[Finding]] + def build_execution_event( + self, event: K8sTriggerEvent, sink_findings: Dict[str, List[Finding]], build_context: Dict[str, Any] ) -> Optional[ExecutionBaseEvent]: # we can't use self.get_execution_event_type() because for KubernetesAnyAllChangesTrigger we need to filter out # stuff like ConfigMaps and we do that filtering here by checking if there is a real event_class @@ -133,7 +136,14 @@ def _build_execution_event( f"classes for kind {event.k8s_payload.kind} cannot be found. skipping. description {event.k8s_payload.description}" ) return None - (obj, old_obj) = self.__parse_kubernetes_objs(event.k8s_payload) + if build_context and OBJ in build_context.keys() and OLD_OBJ in build_context.keys(): + obj = build_context.get(OBJ) + old_obj = build_context.get(OLD_OBJ) + else: + (obj, old_obj) = self.__parse_kubernetes_objs(event.k8s_payload) + build_context[OBJ] = obj + build_context[OLD_OBJ] = old_obj + operation_type = K8sOperationType(event.k8s_payload.operation) return event_class( sink_findings=sink_findings, diff --git a/src/robusta/integrations/prometheus/trigger.py b/src/robusta/integrations/prometheus/trigger.py index ddc322e5c..d5bbfdada 100644 --- a/src/robusta/integrations/prometheus/trigger.py +++ b/src/robusta/integrations/prometheus/trigger.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, List, NamedTuple, Optional, Type, Union +from typing import Any, Dict, List, NamedTuple, Optional, Type, Union from hikaru.model.rel_1_26 import DaemonSet, HorizontalPodAutoscaler, Job, Node, NodeList, StatefulSet from pydantic.main import BaseModel @@ -12,6 +12,8 @@ from robusta.integrations.prometheus.models import PrometheusAlert, PrometheusKubernetesAlert from robusta.utils.cluster_provider_discovery import cluster_provider +ALERT_EVENT = "alert_event" + class PrometheusTriggerEvent(TriggerEvent): alert: PrometheusAlert @@ -63,7 +65,7 @@ class PrometheusAlertTrigger(BaseTrigger): def get_trigger_event(self): return PrometheusTriggerEvent.__name__ - def should_fire(self, event: TriggerEvent, playbook_id: str): + def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict[str, Any]): if not isinstance(event, PrometheusTriggerEvent): return False @@ -91,10 +93,13 @@ def should_fire(self, event: TriggerEvent, playbook_id: str): return True - def _build_execution_event( - self, event: PrometheusTriggerEvent, sink_findings: Dict[str, List[Finding]] + def build_execution_event( + self, event: PrometheusTriggerEvent, sink_findings: Dict[str, List[Finding]], build_context: Dict[str, Any] ) -> Optional[ExecutionBaseEvent]: - return AlertEventBuilder.build_event(event, sink_findings) + if ALERT_EVENT not in build_context.keys(): + build_context[ALERT_EVENT] = AlertEventBuilder.build_event(event, sink_findings) + + return build_context.get(ALERT_EVENT) @staticmethod def get_execution_event_type() -> type: diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 5b0cfa7f3..8b5f8353f 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -250,7 +250,13 @@ def __prepare_runtime_config( # Order matters. Internal playbooks, should be added first, and run first active_playbooks = [ PlaybookDefinition( - triggers=[{"on_kubernetes_any_resource_all_changes": {}}], + triggers=[ + { + "on_kubernetes_resource_operation": { + "resources": ["deployment", "replicaset", "daemonset", "statefulset", "pod", "job"] + } + } + ], actions=[{"cluster_discovery_updates": {}}], ) ] From ad4dfb47d47e43aff17911e89ba81131e775c13c Mon Sep 17 00:00:00 2001 From: Arik Alon Date: Thu, 21 Dec 2023 01:32:49 +0200 Subject: [PATCH 3/3] CR fixes --- .../core/discovery/top_service_resolver.py | 7 ------- .../core/playbooks/internal/discovery_events.py | 17 ++++++++--------- src/robusta/core/triggers/custom_triggers.py | 2 +- ...es_trigger.py => multi_resources_trigger.py} | 0 4 files changed, 9 insertions(+), 17 deletions(-) rename src/robusta/core/triggers/{multi_resoources_trigger.py => multi_resources_trigger.py} (100%) diff --git a/src/robusta/core/discovery/top_service_resolver.py b/src/robusta/core/discovery/top_service_resolver.py index dd6b9e4bc..f073958f4 100644 --- a/src/robusta/core/discovery/top_service_resolver.py +++ b/src/robusta/core/discovery/top_service_resolver.py @@ -74,10 +74,3 @@ def add_cached_resource(cls, resource: TopLevelResource): cls.__recent_resource_updates[resource.get_resource_key()] = CachedResourceInfo( resource=resource, event_time=time.time() ) - - @classmethod - def remove_cached_resource(cls, resource: TopLevelResource): - if resource in cls.__namespace_to_resource[resource.namespace]: - cls.__namespace_to_resource[resource.namespace].remove(resource) - with cls.__cached_updates_lock: - cls.__recent_resource_updates.pop(resource.get_resource_key(), None) diff --git a/src/robusta/core/playbooks/internal/discovery_events.py b/src/robusta/core/playbooks/internal/discovery_events.py index 9b0ba6066..0526facbd 100644 --- a/src/robusta/core/playbooks/internal/discovery_events.py +++ b/src/robusta/core/playbooks/internal/discovery_events.py @@ -20,18 +20,17 @@ @action def cluster_discovery_updates(event: KubernetesAnyChangeEvent): if ( - event.obj.kind in ["Deployment", "ReplicaSet", "DaemonSet", "StatefulSet", "Pod", "Job"] + event.operation in [K8sOperationType.CREATE, K8sOperationType.UPDATE] + and event.obj.kind in ["Deployment", "ReplicaSet", "DaemonSet", "StatefulSet", "Pod", "Job"] and not event.obj.metadata.ownerReferences ): - resource = TopLevelResource( - name=event.obj.metadata.name, - resource_type=event.obj.kind, - namespace=event.obj.metadata.namespace, + TopServiceResolver.add_cached_resource( + TopLevelResource( + name=event.obj.metadata.name, + resource_type=event.obj.kind, + namespace=event.obj.metadata.namespace, + ) ) - if event.operation in [K8sOperationType.CREATE, K8sOperationType.UPDATE]: - TopServiceResolver.add_cached_resource(resource) - elif event.operation == K8sOperationType.DELETE: - TopServiceResolver.remove_cached_resource(resource) @action diff --git a/src/robusta/core/triggers/custom_triggers.py b/src/robusta/core/triggers/custom_triggers.py index 2425ac178..88982c33e 100644 --- a/src/robusta/core/triggers/custom_triggers.py +++ b/src/robusta/core/triggers/custom_triggers.py @@ -10,7 +10,7 @@ WarningEventUpdateTrigger, ) from robusta.core.triggers.job_failed_trigger import JobFailedTrigger -from robusta.core.triggers.multi_resoources_trigger import MultiResourceTrigger +from robusta.core.triggers.multi_resources_trigger import MultiResourceTrigger from robusta.core.triggers.pod_crash_loop_trigger import PodCrashLoopTrigger from robusta.core.triggers.pod_image_pull_backoff import PodImagePullBackoffTrigger from robusta.core.triggers.pod_oom_killed_trigger import PodOOMKilledTrigger diff --git a/src/robusta/core/triggers/multi_resoources_trigger.py b/src/robusta/core/triggers/multi_resources_trigger.py similarity index 100% rename from src/robusta/core/triggers/multi_resoources_trigger.py rename to src/robusta/core/triggers/multi_resources_trigger.py