-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
System Test Framework based on
unittest
Implement a new version of the system test framework. It is based on Python's `unittest` framework. Add requirements for the system test framework. Add a `README.md` to explain these requirements and to give context.
- Loading branch information
Showing
65 changed files
with
838 additions
and
342 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| req.Software_Requirement { | ||
| description = description | ||
| tags = derived_from | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| trlc>=2.0.1 | ||
| requests>=2.31.0 | ||
| libcst>=1.1.0 | ||
| PyYAML>=6.0.2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,3 @@ | ||
| !**/expected-output/* | ||
| !lobster-html-report/**/input/*.lobster | ||
| !lobster-online-report/**/input/*.lobster | ||
| !lobster-online-report/**/input/*.lobster |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| from subprocess import CompletedProcess | ||
| from unittest import TestCase | ||
| from .testrunner import TestRunner | ||
|
|
||
|
|
||
| # pylint: disable=invalid-name | ||
|
|
||
|
|
||
| # Setting this flag will tell unittest not to print tracebacks from this frame. | ||
| # This way our custom assertions will show the interesting line number from the caller | ||
| # frame, and not from this boring file. | ||
| __unittest = True | ||
|
|
||
|
|
||
| class Asserter: | ||
| def __init__(self, system_test_case: TestCase, completed_process: CompletedProcess, | ||
| test_runner: TestRunner): | ||
| self._test_case = system_test_case | ||
| self._completed_process = completed_process | ||
| self._test_runner = test_runner | ||
|
|
||
| def assertExitCode(self, expected: int, msg="Exit code differs"): | ||
| # lobster-trace: system_test.Compare_Exit_Code | ||
| self._test_case.assertEqual( | ||
| expected, | ||
| self._completed_process.returncode, | ||
| msg, | ||
| ) | ||
|
|
||
| def assertStdOutText(self, expected: str, msg="STDOUT differs"): | ||
| # lobster-trace: system_test.Compare_Stdout | ||
| self._test_case.assertEqual(expected, self._completed_process.stdout, msg) | ||
|
|
||
| def assertStdErrText(self, expected: str, msg="STDERR differs"): | ||
| # lobster-trace: system_test.Compare_Stderr | ||
| self._test_case.assertEqual(expected, self._completed_process.stderr, msg) | ||
|
|
||
| def assertNoStdErrText(self, msg="STDERR contains output"): | ||
| self.assertStdErrText("", msg) | ||
|
|
||
| def assertOutputFiles(self): | ||
| """For each expected file, checks if an actual file has been created with the | ||
| expected content | ||
| Before comparing the actual text with the expected text, we do the | ||
| following replacements: | ||
| a) Replace Windows-like slashes \\ with / in order to be able to | ||
| compare the actual output on all OS against the expected output on | ||
| Linux | ||
| b) Replace the fixed string TEST_CASE_PATH with the absolute path to | ||
| the current working directory. This is necessary for tools like | ||
| lobster-cpptest which write absolute paths into their *.lobster | ||
| output files. | ||
| """ | ||
| # lobster-trace: system_test.Compare_Output_Files | ||
| for expected_file_ref in self._test_runner.tool_output_files: | ||
| expected_location = self._test_runner.working_dir / expected_file_ref.name | ||
| try: | ||
| with open( | ||
| expected_file_ref, | ||
| "r", | ||
| encoding="UTF-8", | ||
| ) as expected_file: | ||
| try: | ||
| with open( | ||
| expected_location, | ||
| "r", | ||
| encoding="UTF-8", | ||
| ) as actual_file: | ||
| # lobster-trace: system_test.Slashes | ||
| modified_actual = actual_file.read().replace("\\\\", "/") | ||
|
|
||
| # lobster-trace: system_test.CWD_Placeholder | ||
| modified_expected = expected_file.read().replace( | ||
| "CURRENT_WORKING_DIRECTORY", | ||
| str(self._test_runner.working_dir), | ||
| ) | ||
| self._test_case.assertEqual( | ||
| modified_actual, | ||
| modified_expected, | ||
| f"File differs from expectation {expected_file_ref}!", | ||
| ) | ||
| except FileNotFoundError: | ||
| self._test_case.fail(f"File {expected_file_ref} was not " | ||
| f"generated by the tool under test!") | ||
| except FileNotFoundError as ex: | ||
| self._test_case.fail(f"Invalid test setup: {ex}") |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| !*.lobster |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 3 additions & 3 deletions
6
...-extension/expected-output/output.lobster → ...s-system/lobster-json/data/banana.lobster
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.