Merge pull request #317 from fandango-fuzzer/andreas

More check and warnings, minor bug fixes

.gitignore

@@ -74,9 +74,6 @@ instance/
 # Sphinx documentation
 docs/_build/
 
-# Own test results
-tests/test-marker.txt
-
 # PyBuilder
 .pybuilder/
 target/
@@ -176,3 +173,8 @@ gen/*
 *.o
 utils/lextab.py
 utils/yacctab.py
+
+# Test result markers
+tests/test-marker.txt
+evaluation/test-evaluation.txt
+evaluation/experiments/test-experiments.txt

Makefile

@@ -164,11 +164,45 @@ clean-docs:
 
 ## Tests
 TESTS = tests
-TEST_SOURCES = $(wildcard $(TESTS)/*.py $(TESTS)/resources/*)
+TEST_SOURCES = $(wildcard $(TESTS)/*.py $(TESTS)/resources/* $(TESTS)/docs/*.fan)
+TEST_MARKER = $(TESTS)/test-marker.txt
 
-.PHONY: test tests
-test tests:
-	$(PYTEST) 
+.PHONY: test tests run-tests
+test tests $(TEST_MARKER): $(PYTHON_SOURCES) $(TEST_SOURCES)
+	$(PYTEST)
+	echo 'Success' > $(TEST_MARKER)
+
+run-tests: $(TEST_MARKER)
+
+## Evaluation
+EVALUATION = evaluation
+EVALUATION_SOURCES = $(wildcard $(EVALUATION)/*.py $(EVALUATION)/*/*.py $(EVALUATION)/*/*/*.py $(EVALUATION)/*/*/*.fan $(EVALUATION)/*/*/*.txt)
+EVALUATION_MARKER = $(EVALUATION)/test-evaluation.txt
+
+# python -m evaluation.vs_isla.run_evaluation
+.PHONY: evaluation evaluate
+evaluation evaluate $(EVALUATION_MARKER): $(PYTHON_SOURCES) $(EVALUATION_SOURCES)
+	$(PYTHON) -m evaluation.vs_isla.run_evaluation 1
+	echo 'Success' > $(EVALUATION_MARKER)
+
+run-evaluation: $(EVALUATION_MARKER)
+
+## Experiments
+EXPERIMENTS = $(EVALUATION)/experiments
+EXPERIMENTS_SOURCES = $(wildcard $(EXPERIMENTS)/*/*.py $(EXPERIMENTS)/*/*.fan)
+EXPERIMENTS_MARKER = $(EXPERIMENTS)/test-experiments.txt
+
+.PHONY: experiment experiments
+experiment experiments $(EXPERIMENTS_MARKER): $(PYTHON_SOURCES) $(EXPERIMENTS_SOURCES)
+	$(PYTHON) -m evaluation.experiments.run_experiments
+	echo 'Success' > $(EXPERIMENTS_MARKER)
+
+run-experiments: $(EXPERIMENTS_MARKER)
+
+## All
+.PHONY: run-all
+run-all: $(TEST_MARKER) $(EVALUATION_MARKER) $(EXPERIMENTS_MARKER)
+	@echo 'All tests passed.'
 
 ## Installation
 .PHONY: install install-test install-tests
@@ -185,10 +219,3 @@ install-test install-tests:
 uninstall:
 	$(PIP) uninstall fandango-fuzzer -y
 
-# python -m evaluation.vs_isla.run_evaluation
-.PHONY: evaluation evaluate experiment experiments
-evaluate evaluation:
-	$(PYTHON) -m evaluation.vs_isla.run_evaluation 1
-
-experiment experiments:
-	$(PYTHON) -m evaluation.experiments.run_experiments

docs/Bits.md

@@ -62,64 +62,52 @@ assert _exit_code == 0
 The combination of `--format=bits` and `--start-symbol` is particularly useful to debug bit fields.
 ```
 
-Internally, Fandango treats the individual flags as if they were strings - that is, `"\x00"` for zero bits and `"\x01"` for nonzero bits.
-Hence, we can also apply _constraints_ to the individual flags:
+Internally, Fandango treats individual flags as integers, too.
+Hence, we can also apply _constraints_ to the individual flags.
+For instance, we can profit from the fact that Python treats `0` as False and `1` as True:
 
 ```shell
-$ fandango fuzz --format=bits -f bits.fan -n 10 -c '<italic> == "\x01" and <bold> == "\x00"'
+$ fandango fuzz --format=bits -f bits.fan -n 10 -c '<italic> and <bold>'
 ```
 
 ```{code-cell}
 :tags: ["remove-input"]
-!fandango fuzz --format=bits -f bits.fan -n 10 -c '<italic> == "\x01" and <bold> == "\x00"' --validate
+!fandango fuzz --format=bits -f bits.fan -n 10 -c '<italic> and <bold>' --validate
 assert _exit_code == 0
 ```
 
-This alternative, using `chr()` to generate a single byte out of the specified integer might be more readable:
+Fandango strictly follows a "left-to-right" order - that is, the order in which bits and bytes are specified in the grammar, the most significant bit is stored first.
 
-```shell
-$ fandango fuzz --format=bits -f bits.fan -n 1 -c '<italic> == chr(1) and <bold> == chr(0)'
-```
-
-```{code-cell}
-:tags: ["remove-input"]
-!fandango fuzz --format=bits -f bits.fan -n 1 -c '<italic> == chr(1) and <bold> == chr(0)' --validate
-assert _exit_code == 0
-```
-
-We can also easily set the value of the entire `format_flag` field using a constraint:
+Hence, we can also easily set the value of the entire `brightness` field using a constraint:
 
 ```shell
-$ fandango fuzz --format=bits -f bits.fan -n 1 -c '<format_flag> == chr(0b11110000)'
+$ fandango fuzz --format=bits -f bits.fan -n 1 -c '<brightness> == 0b1111'
 ```
 
 ```{code-cell}
 :tags: ["remove-input"]
-!fandango fuzz --format=bits -f bits.fan -n 1 -c '<format_flag> == chr(0b11110000)' --validate
+!fandango fuzz --format=bits -f bits.fan -n 1 -c '<brightness> == 0b1111' --validate
 assert _exit_code == 0
 ```
 
-Since Fandango strictly follows a "left-to-right" order - that is, the order in which bits and bytes are specified in the grammar, the most significant bit is stored first.
-Thus, the order of bits in the `chr()` argument is identical to the order of bits in the produced output.
-
 ```{note}
 Fandango always strictly follows a "left-to-right" order - that is, the order in which bits and bytes are specified in the grammar.
 ```
 
-To convert a bit into a numerical value, applying the Python `ord()` function comes in handy.
-Note that its argument (the symbol) must be converted into a string first:
+Of course, we can also give the number in decimal format:
 
 ```shell
-$ fandango fuzz --format=bits -f bits.fan -n 1 -c 'ord(str(<brightness>)) > 10'
+$ fandango fuzz --format=bits -f bits.fan -n 1 -c '<brightness> == 15'
 ```
 
+% TODO: This does not work with <format_flag> :-(
 ```{code-cell}
 :tags: ["remove-input"]
-!fandango fuzz --format=bits -f bits.fan -n 10 -c 'ord(str(<brightness>)) > 10' --validate
+!fandango fuzz --format=bits -f bits.fan -n 10 -c '<brightness> == 15' --validate --population-size=20
 assert _exit_code == 0
 ```
 
-Note how the last four bits (the `<brightness>` field) always represent a number greater than ten.
+Note how the last four bits (the `<brightness>` field) are always set to `1111` - the number 15.
 
 ```{warning}
 When implementing a format, be sure to follow its conventions regarding
@@ -140,7 +128,7 @@ $ echo -n '\xf0' | fandango parse -f bits.fan -o - --format=bits
 
 ```{code-cell}
 :tags: ["remove-input"]
-!echo -n '\xf0' | fandango parse -f bits.fan -o - --format=bits
+!echo -n '\xf0' | fandango parse -f bits.fan -o - --format=bits --validate
 assert _exit_code == 0
 ```
 
@@ -174,7 +162,7 @@ $ echo -n '\xf0' | fandango parse -f bits.fan -o - --format=grammar
 
 ```{code-cell}
 :tags: ["remove-input"]
-!echo -n '\xf0' | fandango parse -f bits.fan -o - --format=grammar
+!echo -n '\xf0' | fandango parse -f bits.fan -o - --format=grammar --validate
 assert _exit_code == 0
 ```
 

docs/gif89a.fan

@@ -1,8 +1,8 @@
 
 include('gif.fan')
 
-where <GifHeader>..<Signature> == "GIF"
-where <GifHeader>..<Version> == "89a"
+where <GifHeader>..<Signature> == b"GIF"
+where <GifHeader>..<Version> == b"89a"
 
 <char> ::= <byte>
 <unsigned_short> ::= <byte> <byte>

evaluation/experiments/pixels/pixels.fan

@@ -1,15 +1,14 @@
-<start> ::= <img> ;
-<img> ::= <width> <height> <pixels> ;
-<width> ::= <uint16> ;
-<height> ::= <uint16> ;
-<pixels> ::= <rgb>* ;
-<uint16> ::= <byte> <byte> ;
-<rgb> ::= <byte> <byte> <byte> ;
-<byte> ::= <bit> <bit> <bit> <bit> <bit> <bit> <bit> <bit> ;
-<bit> ::= "0" | "1" ;
-
-int(<pixels>) == int(<width>) * int(<height>) * 3;
-
-
-
-int(<start>) != 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000;
+<start> ::= <img>
+<img> ::= <width> <height> <pixels>
+<width> ::= <uint16>
+<height> ::= <uint16>
+<pixels> ::= <rgb>*
+<uint16> ::= <byte> <byte>
+<rgb> ::= <byte> <byte> <byte>
+<byte> ::= br"[\x00-\xff]"
+
+def uint16(tree):
+    b = tree.to_bytes()
+    return b[1] * 256 + b[0]
+
+where len(<pixels>) == uint16(<width>) * uint16(<height>) * 3

language/FandangoLexer.g4

@@ -18,7 +18,7 @@ INTEGER: DECIMAL_INTEGER | OCT_INTEGER | HEX_INTEGER | BIN_INTEGER;
 
 PYTHON_START: '<py>' {self.python_start()};
 PYTHON_END  : '</py>' {self.python_end()};
-NONTERMINAL: '<' ID_CONTINUE+ '>';
+NONTERMINAL: '<' NAME '>';
 
 // python keywords
 AND        : 'and';

src/fandango/cli/__init__.py

@@ -261,9 +261,9 @@ def get_parser(in_command_line=True):
     )
     file_parser.add_argument(
         "--format",
-        choices=["string", "bits", "tree", "repr", "grammar", "none"],
+        choices=["string", "bits", "tree", "grammar", "value", "repr", "none"],
         default="string",
-        help="produce output(s) as string (default), as a bit string, as a derivation tree, in internal representation, as a grammar, or none",
+        help="produce output(s) as string (default), as a bit string, as a derivation tree, as a grammar, as a Python value, in internal representation, or none",
     )
     file_parser.add_argument(
         "--file-mode",
@@ -682,6 +682,8 @@ def convert(s: str) -> str | bytes:
         return convert(tree.to_bits())
     if args.format == "grammar":
         return convert(tree.to_grammar())
+    if args.format == "value":
+        return convert(tree.to_value())
     if args.format == "none":
         return convert("")
 
@@ -939,7 +941,12 @@ def fuzz_command(args):
 
         # Ensure that every generated file can be parsed
         # and returns the same string as the original
-        temp_dir = tempfile.TemporaryDirectory(delete=False)
+        try:
+            temp_dir = tempfile.TemporaryDirectory(delete=False)
+        except TypeError:
+            # Python 3.11 does not know the `delete` argument
+            temp_dir = tempfile.TemporaryDirectory()
+
         args.directory = temp_dir.name
         args.format = "string"
         output_population(population, args, file_mode=file_mode, output_on_stdout=False)
@@ -1281,6 +1288,8 @@ def _complete(text, state):
                 last_status = run(command, args)
         except SystemExit:
             pass
+        except KeyboardInterrupt:
+            pass
 
     return last_status