Update configs

eth-easl · Sep 15, 2024 · fd94317 · fd94317
1 parent 0f9c602
commit fd94317
Showing 1 changed file with 19 additions and 13 deletions.
diff --git a/experiments/huffpost/compare_trigger_policies/run.py b/experiments/huffpost/compare_trigger_policies/run.py
@@ -55,7 +55,7 @@ def construct_periodic_eval_handlers(
             execution_time=execution_time,
             models="matrix",
             strategy=PeriodicEvalStrategyConfig(
-                eval_every="13w",  # once per quarter
+                every="13w",  # once per quarter
                 interval=f"[-{fake_interval}; +{fake_interval}]",
                 start_timestamp=_FIRST_TIMESTAMP,
                 end_timestamp=_LAST_TIMESTAMP,
@@ -66,14 +66,18 @@ def construct_periodic_eval_handlers(
     ]
 
 
-def construct_between_trigger_eval_handler(execution_time: EvalHandlerExecutionTime = "manual") -> EvalHandlerConfig:
-    return EvalHandlerConfig(
-        name="full",
-        execution_time=execution_time,
-        models="active",
-        strategy=BetweenTwoTriggersEvalStrategyConfig(),
-        datasets=["huffpost_kaggle_all"],  # train and test
-    )
+def construct_between_trigger_eval_handler(
+    execution_time: EvalHandlerExecutionTime = "manual",
+) -> list[EvalHandlerConfig]:
+    return [
+        EvalHandlerConfig(
+            name="full",
+            execution_time=execution_time,
+            models="active",
+            strategy=BetweenTwoTriggersEvalStrategyConfig(),
+            datasets=["huffpost_kaggle_all"],  # train and test
+        )
+    ]
 
 
 def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
@@ -103,8 +107,6 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     #         1X: Baselines with PERIODIC_EVAL_INTERVAL, executed with cautious        #
     #              parallelism and post factum evaluation (bottlenecking)              #
     # -------------------------------------------------------------------------------- #
-    # TODO: merge main
-    # TODO: reset datasets in db
     # time baselines
     10: Experiment(
         name="hp-baseline-time",
@@ -114,7 +116,9 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
         ),
         time_triggers={
             schedule: TimeTriggerConfig(every=schedule, start_timestamp=_FIRST_TIMESTAMP)
-            for schedule in reversed(["13w", "1y", "2y", "4y"])  # TODO: add 26w
+            for schedule in (["26w", "1y", "2y"])  # reversed
+            # 0: "26w", "1y", "2y"
+            # 1: "13w", "4y"
         },
         gpu_device="cuda:0",
     ),
@@ -127,7 +131,9 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
         ),
         data_amount_triggers={
             f"{num_samples}": DataAmountTriggerConfig(num_samples=num_samples)
-            for num_samples in reversed([20_000, 10_000, 40_000, 80_000])  # TODO: add 2_000
+            for num_samples in ([10_000, 20_000, 40_000])
+            # 2: 10_000, 20_000, 40_000
+            # 3: 5_000, 80_000
         },
         gpu_device="cuda:1",
     ),