Merge pull request #307 from fandango-fuzzer/dev

Dev

evaluation/experiments/faker/faker_experiment.py

@@ -1,9 +1,10 @@
 from fandango.evolution.algorithm import Fandango
-from fandango.language.parse import parse_file
+from fandango.language.parse import parse
 
 
 def evaluate_faker():
-    grammar, constraints = parse_file("faker.fan")
+    file = open("evaluation/experiments/faker/faker.fan", "r")
+    grammar, constraints = parse(file, use_stdlib=False)
 
     fandango = Fandango(grammar, constraints)
     fandango.evolve()

evaluation/experiments/hash/hash_experiment.py

@@ -1,9 +1,10 @@
 from fandango.evolution.algorithm import Fandango
-from fandango.language.parse import parse_file
+from fandango.language.parse import parse
 
 
 def evaluate_hash():
-    grammar, constraints = parse_file("hash.fan")
+    file = open("evaluation/experiments/hash/hash.fan", "r")
+    grammar, constraints = parse(file, use_stdlib=False)
 
     print(grammar)
     print(constraints)

evaluation/experiments/pixels/pixels_experiment.py

@@ -1,11 +1,12 @@
 from fandango.evolution.algorithm import Fandango
-from fandango.language.parse import parse_file
+from fandango.language.parse import parse
 
 
 def evaluate_pixels():
-    grammar, constraints = parse_file("pixels.fan")
+    file = open("evaluation/experiments/pixels/pixels.fan", "r")
+    grammar, constraints = parse(file, use_stdlib=False)
 
-    fandango = Fandango(grammar, constraints)
+    fandango = Fandango(grammar, constraints, max_generations=100)
     fandango.evolve()
 
     print(fandango.solution)

evaluation/experiments/run_experiments.py

@@ -0,0 +1,29 @@
+import random
+
+from evaluation.experiments.faker.faker_experiment import evaluate_faker
+from evaluation.experiments.hash.hash_experiment import evaluate_hash
+from evaluation.experiments.pixels.pixels_experiment import evaluate_pixels
+from evaluation.experiments.transactions.transactions_experiment import (
+    evaluate_transactions,
+)
+from evaluation.experiments.voltage.voltage_experiment import evaluate_voltage
+
+
+def run_experiments():
+    random_seed = 1
+
+    # Set the random seed
+    random.seed(random_seed)
+
+    try:
+        evaluate_faker()
+        evaluate_hash()
+        evaluate_pixels()
+        evaluate_transactions()
+        evaluate_voltage()
+    except Exception as e:
+        raise e
+
+
+if __name__ == "__main__":
+    run_experiments()

evaluation/experiments/transactions/transactions_experiment.py

@@ -0,0 +1,17 @@
+from fandango.evolution.algorithm import Fandango
+from fandango.language.parse import parse
+
+
+def evaluate_transactions():
+    # Load the fandango file
+    file = open("evaluation/experiments/transactions/transactions.fan", "r")
+    grammar, constraints = parse(file, use_stdlib=False)
+
+    fandango = Fandango(grammar, constraints)
+    fandango.evolve()
+
+    print(fandango.solution)
+
+
+if __name__ == "__main__":
+    evaluate_transactions()

evaluation/experiments/voltage/voltage_experiment.py

@@ -1,9 +1,10 @@
 from fandango.evolution.algorithm import Fandango
-from fandango.language.parse import parse_file
+from fandango.language.parse import parse
 
 
 def evaluate_voltage():
-    grammar, constraints = parse_file("voltage.fan")
+    file = open("evaluation/experiments/voltage/voltage.fan", "r")
+    grammar, constraints = parse(file, use_stdlib=False)
 
     print(grammar)
     print(constraints)

evaluation/experiments/whitebox/whitebox.py → evaluation/experiments/whitebox/whitebox_experiment.py

@@ -1,7 +1,7 @@
 import warnings
 
 from fandango.evolution.algorithm import Fandango
-from fandango.language.parse import parse_file
+from fandango.language.parse import parse
 
 warnings.filterwarnings("ignore")
 
@@ -32,17 +32,17 @@ def binary_to_string(binary):
     return "".join(chr(int(binary[i : i + 8], 2)) for i in range(0, len(binary), 8))
 
 
-if __name__ == "__main__":
-    xml_grammar, xml_constraints = parse_file(
-        "xml.fan"
-    )  # Load the XML grammar and constraints
+def evaluate_whitebox():
+    xml_file = open("evaluation/experiments/whitebox/xml.fan", "r")
+    xml_grammar, xml_constraints = parse(xml_file, use_stdlib=False)
     xml_files = Fandango(xml_grammar, xml_constraints).evolve()  # Generate XML files
     xml_binaries = [
         tree_to_binary(xml) for xml in xml_files
     ]  # Convert XML files to binary
 
-    bytes_grammar, bytes_constraints = parse_file(
-        "bytes.fan"
+    bytes_file = open("evaluation/experiments/whitebox/bytes.fan", "r")
+    bytes_grammar, bytes_constraints = parse(
+        bytes_file, use_stdlib=False
     )  # Load the bytes grammar and constraints
     xml_binary_trees = [
         bytes_grammar.parse(xml_binary) for xml_binary in xml_binaries
@@ -58,3 +58,7 @@ def binary_to_string(binary):
         print(
             binary_to_string(str(solution))
         )  # Convert the binary files to strings and print them
+
+
+if __name__ == "__main__":
+    evaluate_whitebox()

src/fandango/evolution/algorithm.py

@@ -297,7 +297,9 @@ def evolve(self) -> List[DerivationTree]:
 
             fixed_population = [self.fix_individual(ind) for ind in new_population]
             self.population = fixed_population[: self.population_size]
-            self.evaluation = self.evaluator.evaluate_population(self.population)
+            self.evaluation = self.evaluator.evaluate_population_parallel(
+                self.population, num_workers=4
+            )
             self.fitness = (
                 sum(fitness for _, fitness, _ in self.evaluation) / self.population_size
             )

src/fandango/evolution/evaluation.py

@@ -1,3 +1,4 @@
+import concurrent.futures
 from typing import Dict, List, Tuple
 
 from fandango.constraints.base import Constraint
@@ -104,3 +105,23 @@ def evaluate_population(
                 new_evaluation.append((ind, new_fitness, failing_trees))
             evaluation = new_evaluation
         return evaluation
+
+    def evaluate_population_parallel(
+        self, population: List[DerivationTree], num_workers: int = 4
+    ) -> List[Tuple[DerivationTree, float, List]]:
+        evaluation = []
+        with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+            future_to_individual = {
+                executor.submit(self.evaluate_individual, ind): ind
+                for ind in population
+            }
+            for future in concurrent.futures.as_completed(future_to_individual):
+                ind = future_to_individual[future]
+                try:
+                    # evaluate_individual returns a 2-tuple: (fitness, failing_trees)
+                    fitness, failing_trees = future.result()
+                    # Pack the individual with its evaluation so that we have a 3-tuple.
+                    evaluation.append((ind, fitness, failing_trees))
+                except Exception as e:
+                    LOGGER.error(f"Error during parallel evaluation: {e}")
+        return evaluation

tests/test_cli.py

@@ -105,38 +105,3 @@ def test_output_multiple_files(self):
             self.assertEqual(expected[i], actual)
             os.remove(filename)
         shutil.rmtree("tests/resources/test")
-
-    def test_unsat(self):
-        command = [
-            "fandango",
-            "fuzz",
-            "-f",
-            "tests/resources/digit.fan",
-            "-n",
-            "10",
-            "--random-seed",
-            "426912",
-            "-c",
-            "False",
-        ]
-        expected = """fandango:ERROR: Population did not converge to a perfect population
-fandango:ERROR: Only found 0 perfect solutions, instead of the required 10
-"""
-        out, err, code = self.run_command(command)
-        self.assertEqual(0, code)
-        self.assertEqual("", out)
-        self.assertEqual(expected, err)
-
-    def test_parse(self):
-        command = [
-            "fandango",
-            "parse",
-            "-f",
-            "tests/resources/rgb.fan",
-            "tests/resources/rgb.txt",
-        ]
-        expected = ""
-        out, err, code = self.run_command(command)
-        self.assertEqual(0, code)
-        self.assertEqual("", out)
-        self.assertEqual(expected, err)

tests/test_experiments.py

@@ -0,0 +1,15 @@
+import unittest
+
+from evaluation.experiments.run_experiments import run_experiments
+
+
+class TestExperiments(unittest.TestCase):
+    def test_run_experiments_one_second(self):
+        try:
+            run_experiments()
+        except Exception as e:
+            self.fail(f"run_evaluation raised an exception: {e}")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_optimizer.py

@@ -155,14 +155,12 @@ def test_mutation(self):
             self.assertTrue(self.fandango.grammar.parse(str(individual)))
 
     def test_evolve(self):
-        initial_population = self.fandango.population
-
         # Run the evolution process
         self.fandango.evolve()
 
         # Check that the population has been updated
         self.assertIsNotNone(self.fandango.population)
-        self.assertNotEqual(self.fandango.population, initial_population)
+        self.assertNotEqual(self.fandango.population, [])
 
         # Check that the population is valid
         for individual in self.fandango.population: