Merge branch 'main' into robin/thesis/experiment-configs

.github/workflows/workflow.yaml

@@ -56,7 +56,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup mamba
         uses: ./.github/actions/mamba
@@ -72,7 +72,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup mamba
         uses: ./.github/actions/mamba
@@ -220,7 +220,7 @@ jobs:
 
       - name: Upload HTML coverage report
         if: ${{ matrix.compiler.coverage && matrix.build-type == 'Debug' }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-results
           path: ${{github.workspace}}/build/modyn/tests/coverage
@@ -236,7 +236,7 @@ jobs:
       line-coverage: ${{steps.run_main_test_with_coverage.outputs.LINE_COVERAGE}}
       branch-coverage: ${{steps.run_main_test_with_coverage.outputs.BRANCH_COVERAGE}}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           ref: main
 
@@ -315,7 +315,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup base container
         uses: ./.github/actions/base
@@ -335,7 +335,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Debug
@@ -351,7 +351,7 @@ jobs:
       - cpp_build_and_test
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Asan
@@ -368,7 +368,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Tsan
@@ -385,7 +385,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Release

docs/pipeline/triggers/DRIFT_TRIGGER.md

@@ -197,8 +197,8 @@ classDiagram
         int window_size = 10
     }
 
-    class DynamicPercentileThresholdCriterion {
-        float percentile = 0.05
+    class DynamicQuantileThresholdCriterion {
+        float quantile = 0.05
     }
 
     class DynamicRollingAverageThresholdCriterion {
@@ -209,7 +209,7 @@ classDiagram
     DriftDecisionCriterion <|-- ThresholdDecisionCriterion
     DriftDecisionCriterion <|-- DynamicThresholdCriterion
 
-    DynamicThresholdCriterion <|-- DynamicPercentileThresholdCriterion
+    DynamicThresholdCriterion <|-- DynamicQuantileThresholdCriterion
     DynamicThresholdCriterion <|-- DynamicRollingAverageThresholdCriterion
 ```
 
@@ -252,8 +252,8 @@ classDiagram
         +Deque~float~ score_observations
     }
 
-    class DynamicPercentileThresholdPolicy {
-        +DynamicPercentileThresholdCriterion config
+    class DynamicQuantileThresholdPolicy {
+        +DynamicQuantileThresholdCriterion config
         +bool evaluate_decision(float distance)
     }
 
@@ -269,7 +269,7 @@ classDiagram
 
     DriftDecisionPolicy <|-- ThresholdDecisionPolicy
     DriftDecisionPolicy <|-- DynamicDecisionPolicy
-    DynamicDecisionPolicy <|-- DynamicPercentileThresholdPolicy
+    DynamicDecisionPolicy <|-- DynamicQuantileThresholdPolicy
     DynamicDecisionPolicy <|-- DynamicRollingAverageThresholdPolicy
     DriftDecisionPolicy <|-- HypothesisTestDecisionPolicy
 

experiments/yearbook/compare_trigger_policies/run.py

@@ -40,7 +40,7 @@
     EnsembleTriggerConfig,
 )
 from modyn.config.schema.pipeline.trigger.performance.criterion import (
-    DynamicPercentilePerformanceThresholdCriterion,
+    DynamicQuantilePerformanceThresholdCriterion,
     StaticNumberAvoidableMisclassificationCriterion,
     StaticPerformanceThresholdCriterion,
 )
@@ -360,7 +360,7 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
                     for perf_threshold in [0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
                 }
                 | {
-                    f"dynamic-{deviation}": DynamicPercentilePerformanceThresholdCriterion(  # TODO: check if bug is fixed
+                    f"dynamic-{deviation}": DynamicQuantilePerformanceThresholdCriterion(  # TODO: check if bug is fixed
                         metric="Accuracy",
                         deviation=deviation,
                         absolute=False,
@@ -373,7 +373,7 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
                         expected_accuracy=0.95,  # TODO: variable
                         allow_reduction=allow_reduction,
                         avoidable_misclassification_threshold=num_misclassifications,
-                    )
+                    )  # TODO: avg / quantile
                     for num_misclassifications in [100, 200, 500, 1000, 2000, 5000]
                     for allow_reduction in [True, False]
                 }

modyn/config/schema/pipeline/trigger/drift/criterion.py

@@ -16,14 +16,20 @@ class _DynamicThresholdCriterion(ModynBaseModel):
     needs_calibration: Literal[True] = Field(True)
 
 
-class DynamicPercentileThresholdCriterion(_DynamicThresholdCriterion):
-    """Dynamic threshold based on a extremeness percentile of the previous
+class DynamicQuantileThresholdCriterion(_DynamicThresholdCriterion):
+    """Dynamic threshold based on a extremeness quantile of the previous
     distance values."""
 
-    id: Literal["DynamicPercentileThresholdCriterion"] = "DynamicPercentileThresholdCriterion"
-    percentile: float = Field(
+    id: Literal["DynamicQuantileThresholdCriterion"] = "DynamicQuantileThresholdCriterion"
+    quantile: float = Field(
         0.05,
-        description="The percentile that a threshold has to be in to trigger a drift event.",
+        description=(
+            "The quantile that a threshold has to be in to trigger a drift event. "
+            "0.05 will only trigger in the most extreme 5% of cases. Hence the triggering "
+            "threshold is more extreme than 95% of the previous values."
+        ),
+        min=0.0,
+        max=1.0,
     )
 
 
@@ -42,9 +48,9 @@ class DynamicRollingAverageThresholdCriterion(_DynamicThresholdCriterion):
     )
 
 
-DynamicThresholdCriterion = DynamicPercentileThresholdCriterion | DynamicRollingAverageThresholdCriterion
+DynamicThresholdCriterion = DynamicQuantileThresholdCriterion | DynamicRollingAverageThresholdCriterion
 
 DriftDecisionCriterion = Annotated[
-    ThresholdDecisionCriterion | DynamicPercentileThresholdCriterion | DynamicRollingAverageThresholdCriterion,
+    ThresholdDecisionCriterion | DynamicQuantileThresholdCriterion | DynamicRollingAverageThresholdCriterion,
     Field(discriminator="id"),
 ]

modyn/config/schema/pipeline/trigger/performance/criterion.py

@@ -36,16 +36,18 @@ class _DynamicPerformanceThresholdCriterion(_PerformanceThresholdCriterion):
     needs_calibration: Literal[True] = Field(True)
 
 
-class DynamicPercentilePerformanceThresholdCriterion(_DynamicPerformanceThresholdCriterion):
-    """Dynamic threshold based on a extremeness percentile of the previous
+class DynamicQuantilePerformanceThresholdCriterion(_DynamicPerformanceThresholdCriterion):
+    """Dynamic threshold based on a extremeness quantile of the previous
     distance values."""
 
-    id: Literal["DynamicPercentilePerformanceThresholdCriterion"] = Field(
-        "DynamicPercentilePerformanceThresholdCriterion"
-    )
-    percentile: float = Field(
+    id: Literal["DynamicQuantilePerformanceThresholdCriterion"] = Field("DynamicQuantilePerformanceThresholdCriterion")
+    quantile: float = Field(
         0.05,
-        description="The percentile that a threshold has to be in to trigger a drift event.",
+        description=(
+            "The quantile that a threshold has to trigger. "
+            "0.05 will only trigger in the most extreme 5% of cases. Hence the triggering "
+            "threshold is more extreme than 95% of the previous values."
+        ),
     )
 
 
@@ -116,7 +118,7 @@ class StaticNumberAvoidableMisclassificationCriterion(_NumberAvoidableMisclassif
 
 PerformanceTriggerCriterion = Annotated[
     StaticPerformanceThresholdCriterion
-    | DynamicPercentilePerformanceThresholdCriterion
+    | DynamicQuantilePerformanceThresholdCriterion
     | DynamicRollingAveragePerformanceThresholdCriterion
     | StaticNumberAvoidableMisclassificationCriterion,
     Field(discriminator="id"),

modyn/supervisor/internal/triggers/datadrifttrigger.py

@@ -7,7 +7,7 @@
 
 from modyn.config.schema.pipeline import DataDriftTriggerConfig
 from modyn.config.schema.pipeline.trigger.drift.criterion import (
-    DynamicPercentileThresholdCriterion,
+    DynamicQuantileThresholdCriterion,
     DynamicRollingAverageThresholdCriterion,
     ThresholdDecisionCriterion,
 )
@@ -42,7 +42,7 @@
 )
 from modyn.supervisor.internal.triggers.utils.decision_policy import (
     DecisionPolicy,
-    DynamicPercentileThresholdPolicy,
+    DynamicQuantileThresholdPolicy,
     DynamicRollingAverageThresholdPolicy,
     StaticThresholdDecisionPolicy,
 )
@@ -366,10 +366,10 @@ def _setup_decision_policies(
             policies[metric_name] = StaticThresholdDecisionPolicy(
                 threshold=criterion.threshold, triggering_direction="higher"
             )
-        elif isinstance(criterion, DynamicPercentileThresholdCriterion):
-            policies[metric_name] = DynamicPercentileThresholdPolicy(
+        elif isinstance(criterion, DynamicQuantileThresholdCriterion):
+            policies[metric_name] = DynamicQuantileThresholdPolicy(
                 window_size=criterion.window_size,
-                percentile=criterion.percentile,
+                quantile=criterion.quantile,
                 triggering_direction="higher",
             )
         elif isinstance(criterion, DynamicRollingAverageThresholdCriterion):

modyn/supervisor/internal/triggers/performance/decision_policy.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 
 from modyn.config.schema.pipeline.trigger.performance.criterion import (
-    DynamicPercentilePerformanceThresholdCriterion,
+    DynamicQuantilePerformanceThresholdCriterion,
     DynamicRollingAveragePerformanceThresholdCriterion,
     StaticNumberAvoidableMisclassificationCriterion,
     StaticPerformanceThresholdCriterion,
@@ -17,7 +17,7 @@
     PerformanceTracker,
 )
 from modyn.supervisor.internal.triggers.utils.decision_policy import (
-    DynamicPercentileThresholdPolicy,
+    DynamicQuantileThresholdPolicy,
     DynamicRollingAverageThresholdPolicy,
     StaticThresholdDecisionPolicy,
 )
@@ -79,17 +79,17 @@ def evaluate_decision(
         return self._wrapped.evaluate_decision(measurement=evaluation_scores[self.metric])
 
 
-class DynamicPerformancePercentileThresholdPolicy(PerformanceDecisionPolicy):
+class DynamicPerformanceQuantileThresholdPolicy(PerformanceDecisionPolicy):
     """Wrapper for DynamicRollingAverageThresholdPolicy.
 
-    Triggers if value is in the lower percentile of the rolling window.
+    Triggers if value is in the lower quantile of the rolling window.
     """
 
-    def __init__(self, config: DynamicPercentilePerformanceThresholdCriterion):
+    def __init__(self, config: DynamicQuantilePerformanceThresholdCriterion):
         self.metric = config.metric
-        self._wrapped = DynamicPercentileThresholdPolicy(
+        self._wrapped = DynamicQuantileThresholdPolicy(
             window_size=config.window_size,
-            percentile=config.percentile,
+            quantile=config.quantile,
             triggering_direction="lower",
         )
 

modyn/supervisor/internal/triggers/performancetrigger.py

@@ -5,7 +5,7 @@
 from typing_extensions import override
 
 from modyn.config.schema.pipeline.trigger.performance.criterion import (
-    DynamicPercentilePerformanceThresholdCriterion,
+    DynamicQuantilePerformanceThresholdCriterion,
     DynamicRollingAveragePerformanceThresholdCriterion,
     StaticNumberAvoidableMisclassificationCriterion,
     StaticPerformanceThresholdCriterion,
@@ -15,7 +15,7 @@
 )
 from modyn.supervisor.internal.triggers.batchedtrigger import BatchedTrigger
 from modyn.supervisor.internal.triggers.performance.decision_policy import (
-    DynamicPerformancePercentileThresholdPolicy,
+    DynamicPerformanceQuantileThresholdPolicy,
     DynamicPerformanceRollingAverageThresholdPolicy,
     PerformanceDecisionPolicy,
     StaticNumberAvoidableMisclassificationDecisionPolicy,
@@ -157,8 +157,8 @@ def _setup_decision_policies(
     for name, criterion in config.decision_criteria.items():
         if isinstance(criterion, StaticPerformanceThresholdCriterion):
             policies[name] = StaticPerformanceThresholdDecisionPolicy(criterion)
-        elif isinstance(criterion, DynamicPercentilePerformanceThresholdCriterion):
-            policies[name] = DynamicPerformancePercentileThresholdPolicy(criterion)
+        elif isinstance(criterion, DynamicQuantilePerformanceThresholdCriterion):
+            policies[name] = DynamicPerformanceQuantileThresholdPolicy(criterion)
         elif isinstance(criterion, DynamicRollingAveragePerformanceThresholdCriterion):
             policies[name] = DynamicPerformanceRollingAverageThresholdPolicy(criterion)
         elif isinstance(criterion, StaticNumberAvoidableMisclassificationCriterion):