remove pytest-depends, rerouting functions (#250)

agbenchmark/conftest.py

@@ -14,7 +14,6 @@
     generate_combined_suite_report,
     generate_single_call_report,
     session_finish,
-    setup_dummy_dependencies,
 )
 from agbenchmark.start_benchmark import CONFIG_PATH, get_regression_data
 from agbenchmark.utils.data_types import SuiteConfig
@@ -23,6 +22,8 @@
     1500  # The tests will stop after 25 minutes so we can send the reports.
 )
 
+pytest_plugins = ["agbenchmark.utils.dependencies"]
+
 
 def resolve_workspace(workspace: str) -> str:
     if workspace.startswith("${") and workspace.endswith("}"):
@@ -159,15 +160,12 @@ def pytest_runtest_makereport(item: Any, call: Any) -> None:
     flags = "--test" in sys.argv or "--maintain" in sys.argv or "--improve" in sys.argv
 
     if call.when == "call":
-        test_name = ""
         # if it's a same task suite, we combine the report.
         # but not if it's a single --test
         if is_suite and is_suite.same_task and not flags:
-            test_name = is_suite.prefix
             generate_combined_suite_report(item, challenge_data, challenge_location)
         else:
             # single non suite test
-            test_name = challenge_data["name"]
             generate_single_call_report(item, call, challenge_data)
         # else: it's a same_task=false suite (tests aren't combined)
     if call.when == "teardown":
@@ -204,16 +202,6 @@ def scores(request: Any) -> None:
     return request.node.cls.scores.get(test_class_name)
 
 
-def pytest_generate_tests(metafunc: Any) -> None:
-    """This is to generate the dummy dependencies each test class"""
-    test_class_instance = metafunc.cls()
-
-    if test_class_instance.setup_dependencies:
-        test_class = metafunc.cls
-        setup_dummy_dependencies(test_class_instance, test_class)
-        setattr(test_class, "setup_dependencies", [])
-
-
 # this is adding the dependency marker and category markers automatically from the json
 def pytest_collection_modifyitems(items: Any, config: Any) -> None:
     data = get_regression_data()
@@ -222,7 +210,6 @@ def pytest_collection_modifyitems(items: Any, config: Any) -> None:
         # Assuming item.cls is your test class
         test_class_instance = item.cls()
 
-        # if it's a dummy dependency setup test, we also skip
         if "test_method" not in item.name:
             continue
 
@@ -231,28 +218,22 @@ def pytest_collection_modifyitems(items: Any, config: Any) -> None:
         dependencies = test_class_instance.data.dependencies
 
         # Filter dependencies if they exist in regression data if its an improvement test
-        if (
-            config.getoption("--improve")
-            or config.getoption("--category")
-            or test_class_instance.setup_dependencies  # same_task suite
-        ):
+        if config.getoption("--improve") or config.getoption(
+            "--category"
+        ):  # TODO: same task suite
             dependencies = [dep for dep in dependencies if not data.get(dep, None)]
-        if (
+        if (  # TODO: separate task suite
             config.getoption("--test")
-            or (  # separate task suite
-                not test_class_instance.setup_dependencies
-                and config.getoption("--suite")
-            )
             or config.getoption("--no_dep")
             or config.getoption("--maintain")
         ):
             dependencies = []
 
-        categories = test_class_instance.data.category
-
         # Add depends marker dynamically
         item.add_marker(pytest.mark.depends(on=dependencies, name=name))
 
+        categories = test_class_instance.data.category
+
         # Add category marker dynamically
         for category in categories:
             item.add_marker(getattr(pytest.mark, category))

agbenchmark/generate_test.py

@@ -5,7 +5,7 @@
 import types
 from collections import deque
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Callable, Dict, Optional
 
 import pytest
 
@@ -14,35 +14,75 @@
 from agbenchmark.utils.data_types import ChallengeData, SuiteConfig
 from agbenchmark.utils.utils import get_test_path
 
+DATA_CATEGORY = {}
+
+
+def setup_dummy_dependencies(
+    file_datum: list[dict[str, Any]],
+    challenge_class: Any,
+    challenge_data: ChallengeData,
+) -> None:
+    """Sets up the dependencies if it's a suite. Creates tests that pass
+    based on the main test run."""
+
+    def create_test_func(test_name: str) -> Callable[[Any, dict[str, Any]], None]:
+        # This function will return another function
+
+        # Define a dummy test function that does nothing
+        def setup_dependency_test(self: Any, scores: dict[str, Any]) -> None:
+            scores = self.get_dummy_scores(test_name, scores)
+            assert scores == 1
+
+        return setup_dependency_test
+
+    for datum in file_datum:
+        DATA_CATEGORY[datum["name"]] = challenge_data.category[0]
+        test_func = create_test_func(datum["name"])
+        # TODO: replace this once I figure out actual dependencies
+        test_func = pytest.mark.depends(on=[challenge_data.name], name=datum["name"])(
+            test_func
+        )
+        test_func = pytest.mark.parametrize(
+            "challenge_data",
+            [None],
+            indirect=True,
+        )(test_func)
+
+        # Add category markers
+        for category in challenge_data.category:
+            test_func = getattr(pytest.mark, category)(test_func)
+
+        test_func = pytest.mark.usefixtures("scores")(test_func)
+        setattr(challenge_class, f"test_{datum['name']}", test_func)
+
 
 def create_single_test(
     data: Dict[str, Any] | ChallengeData,
     challenge_location: str,
-    suite_config: Optional[SuiteConfig] = None,
+    file_datum: Optional[list[dict[str, Any]]] = None,
 ) -> None:
     challenge_data = None
     artifacts_location = None
     if isinstance(data, ChallengeData):
         challenge_data = data
         data = data.get_data()
 
+    DATA_CATEGORY[data["name"]] = data["category"][0]
+
     # Define test class dynamically
     challenge_class = types.new_class(data["name"], (Challenge,))
 
     clean_challenge_location = get_test_path(challenge_location)
     setattr(challenge_class, "CHALLENGE_LOCATION", clean_challenge_location)
 
-    # if its a parallel run suite we just give it the data
-    if suite_config and suite_config.same_task:
+    # in the case of a suite
+    if isinstance(challenge_data, ChallengeData):
+        if file_datum:  # same task suite
+            setup_dummy_dependencies(file_datum, challenge_class, challenge_data)
+
         artifacts_location = str(Path(challenge_location).resolve())
         if "--test" in sys.argv or "--maintain" in sys.argv or "--improve" in sys.argv:
             artifacts_location = str(Path(challenge_location).resolve().parent.parent)
-        else:
-            setattr(
-                challenge_class,
-                "setup_dependencies",
-                [test_name for test_name in data["info"].keys()],
-            )
         setattr(
             challenge_class,
             "_data_cache",
@@ -83,15 +123,8 @@ def test_method(self, config: Dict[str, Any], request) -> None:  # type: ignore
     setattr(module, data["name"], challenge_class)
 
 
-def create_single_suite_challenge(
-    suite_config: SuiteConfig, data: Dict[str, Any], path: Path
-) -> None:
-    test_data = suite_config.challenge_from_test_data(data)
-    create_single_test(
-        test_data,
-        str(path),
-        suite_config=suite_config,
-    )
+def create_single_suite_challenge(challenge_data: ChallengeData, path: Path) -> None:
+    create_single_test(challenge_data, str(path))
 
 
 def create_challenge(
@@ -106,7 +139,8 @@ def create_challenge(
 
         # if its a single test running we dont care about the suite
         if "--test" in sys.argv or "--maintain" in sys.argv or "--improve" in sys.argv:
-            create_single_suite_challenge(suite_config, data, path)
+            challenge_data = suite_config.challenge_from_test_data(data)
+            create_single_suite_challenge(challenge_data, path)
             return json_files
 
         # Get all data.json files within the grandparent directory
@@ -132,7 +166,7 @@ def create_challenge(
             challenge_data = suite_config.challenge_from_datum(file_datum)
 
             create_single_test(
-                challenge_data, str(grandparent_dir), suite_config=suite_config
+                challenge_data, str(grandparent_dir), file_datum=file_datum
             )
         else:
             reverse = suite_config.reverse_order

agbenchmark/reports/reports.py

@@ -2,9 +2,7 @@
 import os
 import sys
 from pathlib import Path
-from typing import Any, Callable
-
-import pytest
+from typing import Any
 
 from agbenchmark.agent_interface import MOCK_FLAG
 from agbenchmark.reports.ReportManager import ReportManager
@@ -194,41 +192,6 @@ def generate_single_call_report(
     item.info_details = info_details
 
 
-def setup_dummy_dependencies(test_class_instance: Any, test_class: Any) -> None:
-    """Sets up the dependencies if it's a suite. Creates tests that pass
-    based on the main test run."""
-
-    def create_test_func(test_name: str) -> Callable[[Any, dict[str, Any]], None]:
-        # This function will return another function
-
-        # Define a dummy test function that does nothing
-        def setup_dependency_test(self: Any, scores: dict[str, Any]) -> None:
-            scores = self.get_dummy_scores(test_name, scores)
-            assert scores == 1
-
-        return setup_dependency_test
-
-    for test_name in test_class_instance.setup_dependencies:
-        setup_dependency_test = create_test_func(test_name)
-        # Add the dummy test function to the class that the current test is part of
-        # TODO: remove on=[test_class.__name__] and fix the actual dependencies problem
-        test_func = pytest.mark.depends(on=[test_class.__name__], name=test_name)(
-            setup_dependency_test
-        )
-        # Parametrize to tell makereport to skip it
-        test_func = pytest.mark.parametrize(
-            "challenge_data",
-            [None],
-            indirect=True,
-        )(test_func)
-        # Add category markers
-        for category in test_class_instance.data.category:
-            test_func = getattr(pytest.mark, category)(test_func)
-
-        test_func = pytest.mark.usefixtures("scores")(test_func)
-        setattr(test_class, f"test_{test_name}", test_func)
-
-
 def finalize_reports(item: Any, challenge_data: dict[str, Any]) -> None:
     run_time = dict(item.user_properties).get("run_time")
 

agbenchmark/start_benchmark.py

@@ -22,6 +22,7 @@
         "benchmark_start_time", BENCHMARK_START_TIME
     )
 
+
 (
     HOME_DIRECTORY,
     CONFIG_PATH,

agbenchmark/utils/challenge.py

@@ -25,7 +25,6 @@ class Challenge(ABC):
     _data_cache: Dict[str, ChallengeData] = {}
     CHALLENGE_LOCATION: str = ""
     ARTIFACTS_LOCATION: str = ""  # this is for suites
-    setup_dependencies: List[str] = []  # this is for suites
     scores: dict[str, Any] = {}  # this is for suites
 
     @property