feat: evaluation is now part of the test suite

.github/workflows/deploy-book.yml

@@ -35,7 +35,7 @@ jobs:
     # Step 4: Install dependencies
     - name: Install dependencies
       run: |
-        pip install -e ".[evaluation]"
+        pip install -e ".[test]"
         pip install jupyter-book pyppeteer ghp-import black graphviz
 
     # Step 5: Locate Fandango and add to PATH

.github/workflows/python-tests.yml

@@ -44,7 +44,7 @@ jobs:
       # Step 4: Install system-level dependencies required by the project
       - name: Install system dependencies
         run: |
-          sudo apt-get update && sudo apt-get install -y python3-dev graphviz libgraphviz-dev pkg-config
+          sudo apt-get update && sudo apt-get install -y python3-dev graphviz libgraphviz-dev pkg-config libarchive-tools
           pip install pygraphviz
 
       # Step 5: Install Python dependencies using make and run tests

evaluation/vs_isla/run_evaluation.py

@@ -1,4 +1,5 @@
 import random
+import sys
 from typing import Tuple
 
 from evaluation.vs_isla.csv_evaluation.csv_evaluation import evaluate_csv
@@ -28,16 +29,29 @@ def better_print_results(
     print("")
 
 
-def run_evaluation(seconds: int = 3600, random_seed: int = 1):
+def run_evaluation(time: int = 3600):
+    seconds = 3600
+    random_seed = 1
+
+    if time is not None:
+        seconds = int(time)
+        print(f"Running evaluation with a time limit of {seconds} seconds.")
+    else:
+        print("Running evaluation with default settings (1 hour).")
+
     # Set the random seed
     random.seed(random_seed)
 
-    better_print_results(evaluate_csv(seconds))
-    better_print_results(evaluate_rest(seconds))
-    better_print_results(evaluate_scriptsizec(seconds))
-    better_print_results(evaluate_tar(seconds))
-    better_print_results(evaluate_xml(seconds))
+    try:
+        better_print_results(evaluate_csv(seconds))
+        better_print_results(evaluate_rest(seconds))
+        better_print_results(evaluate_scriptsizec(seconds))
+        better_print_results(evaluate_tar(seconds))
+        better_print_results(evaluate_xml(seconds))
+    except Exception as e:
+        raise e
 
 
 if __name__ == "__main__":
-    run_evaluation()
+    arg = sys.argv[1] if len(sys.argv) > 1 else None
+    run_evaluation(arg)

evaluation/vs_isla/uvl_evaluation/uvl.fan

@@ -0,0 +1,3 @@
+<start> ::= <feature_model>;
+
+<feature_model> ::= "init";

evaluation/vs_isla/uvl_evaluation/uvl_evaluation.py

@@ -0,0 +1,56 @@
+import time
+from typing import Tuple
+
+from fandango.evolution.algorithm import Fandango, LoggerLevel
+from fandango.language.parse import parse
+
+
+def is_syntactically_valid_uvl(uvl_string):
+    try:
+        return True
+    except Exception:
+        pass
+
+
+def evaluate_uvl(
+    seconds=60,
+) -> Tuple[str, int, int, float, Tuple[float, int, int], float, float]:
+    file = open("evaluation/vs_isla/uvl_evaluation/uvl.fan", "r")
+    grammar, constraints = parse(file, use_stdlib=False)
+    solutions = []
+
+    time_in_an_hour = time.time() + seconds
+
+    while time.time() < time_in_an_hour:
+        fandango = Fandango(
+            grammar,
+            constraints,
+            desired_solutions=100,
+            logger_level=LoggerLevel.INFO,
+        )
+        fandango.evolve()
+        solutions.extend(fandango.solution)
+
+    coverage = grammar.compute_grammar_coverage(solutions, 4)
+
+    valid = []
+    for solution in solutions:
+        if is_syntactically_valid_uvl(str(solution)):
+            valid.append(solution)
+
+    set_mean_length = sum(len(str(x)) for x in valid) / len(valid)
+    set_medium_length = sorted(len(str(x)) for x in valid)[len(valid) // 2]
+    valid_percentage = len(valid) / len(solutions) * 100
+    return (
+        "UVL",
+        len(solutions),
+        len(valid),
+        valid_percentage,
+        coverage,
+        set_mean_length,
+        set_medium_length,
+    )
+
+
+if __name__ == "__main__":
+    print(evaluate_uvl())

pyproject.toml

@@ -49,10 +49,7 @@ test = [
     "pytest-cov>=4.1.0",
     "pytest-html>=3.2.0",
     "pytest-rerunfailures>=11.1.2",
-    "parameterized>=0.8.1"
-]
-
-evaluation = [
+    "parameterized>=0.8.1",
     "Faker>=30.4.0",
     "docutils>=0.20.1",  # Jupyter-book needs docutils<=0.20
     "tccbox",

tests/test_evaluation.py

@@ -0,0 +1,16 @@
+import unittest
+
+from evaluation.vs_isla.run_evaluation import run_evaluation
+
+
+class TestEvaluation(unittest.TestCase):
+    def test_run_evaluation_one_second(self):
+        # Run the evaluation for 1 second and ensure it doesn't throw exceptions
+        try:
+            run_evaluation(1)  # Run evaluation with time limit of 1 second
+        except Exception as e:
+            self.fail(f"run_evaluation raised an exception: {e}")
+
+
+if __name__ == "__main__":
+    unittest.main()