diff --git a/CHANGELOG.md b/CHANGELOG.md
index dd43cf9..59254c1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [1.6.0] - Oct 09, 2021
+
+### Added
+
+* Add test / tester *IDs* (see https://github.com/qtc-de/tricot/tree/main/docs#selective-testing)
+* Add *test groups* (see https://github.com/qtc-de/tricot/tree/main/docs#selective-testing)
+
+### Changed
+
+* Full help menu is now always displayed on argparse errors
+* Small bugfixes and formatting changes
+
+### Removed
+
+* The `--tester`, `--exclude` and `--number` options were removed.
+ They are replaced by the `--ids`, `--groups`, `--exclude-ids` and
+ `--exclude-groups` options.
+
+
## [1.5.0] - Sep 11, 2021
### Added
diff --git a/bin/tricot b/bin/tricot
index 2b72c71..3802b26 100644
--- a/bin/tricot
+++ b/bin/tricot
@@ -9,22 +9,39 @@ import docker
import tricot
import argparse
+from tricot.constants import VERSION
-parser = argparse.ArgumentParser(description='''tricot v1.5.0 - a trivial command tester that allows you to verify that certain
- commands or executables behave as expected. It uses .yml files for test
- definitions and can be used from the command line or as a python library.''',
- formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=40))
+
+class FullHelpParser(argparse.ArgumentParser):
+ '''
+ Custom ArgumentParser class to show more helpful help messages per default.
+ Taken from: https://stackoverflow.com/a/4042861
+ '''
+ def error(self, message):
+ '''
+ Show the whole help menu per default.
+ '''
+ sys.stderr.write(f'error: {message}\n\n')
+ self.print_help()
+ sys.exit(2)
+
+
+parser = FullHelpParser(description=f'''tricot v{VERSION} - a trivial command tester to verify that commands,
+ scripts or executables behave as expected. It uses .yml files for test
+ definitions and can be used from the command line or as a python library.''',
+ formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=60))
parser.add_argument('--debug', dest='debug', action='store_true', help='enable debug output')
-parser.add_argument('--exclude', metavar='name', nargs='+', default=[], help='exclude the specified testers')
+parser.add_argument('--exclude-ids', dest='eids', metavar='id', nargs='+', default=[], help='exclude the specified test / tester IDs')
+parser.add_argument('--exclude-groups', dest='egroups', metavar='group', nargs='+', default=[], help='exclude the specified test groups')
parser.add_argument('file', metavar='file', nargs='+', help='test definition (.yml file)')
-parser.add_argument('--logfile', dest='log', type=argparse.FileType('w'), help='mirror output into a logfile')
+parser.add_argument('--groups', metavar='group', nargs='+', default=[], help='only run the specified test groups')
+parser.add_argument('--ids', metavar='id', nargs='+', default=[], help='only run the specified test / tester IDs')
+parser.add_argument('--logfile', dest='log', metavar='file', type=argparse.FileType('w'), help='mirror output into a logfile')
parser.add_argument('--load', dest='load', metavar='file', nargs='+', default=[], type=argparse.FileType('r'), help='custom validators, extractors and plugins')
-parser.add_argument('--numbers', dest='numbers', metavar='int', type=int, nargs='+', default=[], help='only run the specified test numbers')
-parser.add_argument('--positionals', dest='pos', metavar='pos', nargs='+', default=[], help='positional variables (accessible by $1, $2, ...)')
+parser.add_argument('--positionals', dest='pos', metavar='var', nargs='+', default=[], help='positional variables (accessible by $1, $2, ...)')
parser.add_argument('-q', '--quite', dest='quite', action='store_true', help='disable verbose output during tests')
parser.add_argument('--template', dest='template', choices=['tester', 'plugin', 'validator', 'extractor'], help='write a template file')
-parser.add_argument('--testers', dest='testers', metavar='name', nargs='+', default=[], help='only run the specified testers')
parser.add_argument('--variables', dest='vars', metavar='vars', nargs='+', default=[], help='runtime variables')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='enable verbose logging during tests')
@@ -152,11 +169,16 @@ def main():
load(args.load)
variables = prepare_variables(args)
+ groups = tricot.utils.parse_groups(args.groups)
+ egroups = tricot.utils.parse_groups(args.egroups)
+
+ tricot.Logger.print_mixed_yellow('tricot', f'v{VERSION}', '- Starting tests...')
+
for yml_file in args.file:
try:
tester = tricot.Tester.from_file(yml_file, runtime_vars=variables)
- tester.run(args.testers, args.numbers, args.exclude)
+ tester.run(set(args.ids), groups, set(args.eids), egroups)
except tricot.ValidatorError as e:
tricot.Logger.print_mixed_red('Caught', 'ValidatorError', 'while parsing test configuration.', e=True)
@@ -249,6 +271,11 @@ def main():
tricot.Logger.print_with_indent_blue(str(e), e=True)
sys.exit(tricot.constants.DOCKER_API)
+ except tricot.DuplicateIDError as e:
+ tricot.Logger.print_mixed_yellow('Caught', 'DuplicateIDError', 'while parsing test configuration.', e=True)
+ tricot.Logger.print_with_indent_blue(str(e), e=True)
+ sys.exit(tricot.constants.DUPLICATE_ID_ERROR)
+
except tricot.TricotRuntimeError as e:
tricot.Logger.print_mixed_blue('Caught', 'unexpected Exception', 'while running the test command.', e=True)
tricot.Logger.print_yellow('Original Error:', e=True)
@@ -256,6 +283,13 @@ def main():
tricot.Logger.print_with_indent_blue(str(e.original), e=True)
sys.exit(tricot.constants.RUNTIME_ERROR)
+ except yaml.scanner.ScannerError as e:
+ tricot.Logger.print_mixed_yellow('Caught', 'yaml.scanner.ScannerError', 'while parsing test configuration.', e=True)
+ tricot.Logger.print('Seems that there is a syntax error within your test configuration.', e=True)
+ tricot.Logger.increase_indent()
+ tricot.Logger.print_with_indent_blue(str(e), e=True)
+ sys.exit(tricot.constants.YAML_SYNTAX_ERROR)
+
except KeyboardInterrupt:
tricot.Logger.reset_indent()
tricot.Logger.print('')
@@ -267,7 +301,7 @@ def main():
if args.debug:
raise e
- tricot.Logger.print_mixed_yellow('Caught', 'unexpected Exception', e=True)
+ tricot.Logger.print_mixed_yellow('Caught', 'unexpected Exception', 'while running tricot.', e=True)
tricot.Logger.increase_indent()
tricot.Logger.print_with_indent_blue(str(e), e=True)
sys.exit(tricot.constants.UNKNOWN)
diff --git a/docs/README.md b/docs/README.md
index 5dd6ce3..9f2c443 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -9,8 +9,9 @@ on it's own.
- [Extractor, Validator and Plugin List](#extractor-validator-and-plugin-list)
- [Writing Custom Plugins](#writing-custom-plugins)
- [Writing Custom Validators](#writing-custom-validators)
-- [Writing Custom Extactors](#writing-custom-extractors)
+- [Writing Custom Extractors](#writing-custom-extractors)
- [Accessing Command Information from an Validator](#accessing-command-information-from-an-validator)
+- [Selective Testing](#selective-testing)
- [Environment Variables](#environment-variables)
- [Runtime Variables](#runtime-variables)
- [Nesting Variables](#nesting-variables)
@@ -354,7 +355,7 @@ plugins:
key2: 22
```
-* ``param_type`` can be used to specify the python type that is expected for the toplevel argument of a
+* ``param_type`` can be used to specify the python type that is expected for the top level argument of a
plugin or validator. In the example test configuration above, the ``param_type`` values should be set
like this:
* ``example_one``: ``param_type = str``
@@ -372,6 +373,108 @@ The parameter validation described above is very basic and has obviously limitat
want a parameter validation that is easier to use and has an arbitrary recursion depth.
+### Selective Testing
+
+----
+
+A common testing scenario is that you just changed a portion of a program and only want to run tests for the affected
+component. *tricot* supports this *selective testing* approach by using *IDs* and *test groups*.
+
+*IDs* are exactly what the name suggests, a unique identifier for each test / tester. You can assign them by using the
+`id` key within of test definitions. *IDs* are ordinary strings and can contain any characters. If a test / tester
+is defined without an *ID*, it's *title* (Test) or *name* (Tester) attribute is used as an *ID*. However, in this case
+*tricot* does not check for duplicate *IDs* and you may end up with multiple tests / testers having the same *ID*.
+
+```yaml
+tester:
+ id: '001'
+ name: Basic Usage
+ description: |-
+ Demonstrate the basic usage of tricot
+
+tests:
+
+ - id: '001-1'
+ title: Test passwd File
+ description: |-
+ ...
+```
+
+To launch tests based on an *ID* you can use the command line switches ``--ids`` and ``--exclude-ids``. When using
+``--ids``, *tricot* only runs the tests / testers that match the specified *IDs*. If the *ID* belongs to a tester,
+all nested testers and tests are run, independent of their *ID*. The ``--exclude-ids`` can be used to exclude certain
+test / tester *IDs* from a test. Notice that ``--exclude-ids`` triggers before ``--ids``, so if you specify the same
+*ID* for both command line options, it is not run. On the other hand, this allows you to exclude nested test / tester
+*IDs* that are contained within a tester specified with the ``--ids`` option.
+
+*Test groups* can be used to group tests / testers together. Each test / tester definition can contain a ``groups`` key,
+which is a list within the *YAML* configuration. The contained items are the groups for the corresponding test / tester.
+*Test groups* are inherited from parent testers, but stacked instead of being merged together. E.g. when a parent tester
+is in the group `io` and the child tester in the group `logging`, the resulting group for the child tester is `io,logging`.
+
+```yaml
+# [io.yml]
+tester:
+ group:
+ - io
+ name: Test IO Modules
+ description: |-
+ Test IO Modules for the Software
+
+testers:
+ - ./nested.yml
+
+
+# [nested.yml]
+tester:
+ group:
+ - logging
+ name: Test IO Modules - logging
+ description: |-
+ Test IO Modules for the Software - logging
+
+tests:
+
+ title: Test Error Log
+ description: |-
+ ...
+```
+
+As for *IDs*, you can use the ``--groups`` and ``--exclude-groups`` command line options to run selective tests on *test
+groups*. However, group specifications on the command line support some special syntax. The easiest case is that you just
+want to run a single test group. E.g. taking the example above, to run the `io,logging` test you could use:
+
+```console
+tricot -v example.yml --groups io,logging
+```
+
+This is straight forwards, but it can get annoying if you defined `logging` groups in other parent testers than `io`.
+To make runs of a single test group, that is contained within different parent test groups easier, it is possible
+to specify wildcards.
+
+* `*` can be used to match an arbitrary group
+* `**` can be used to match an arbitrary number of arbitrary groups
+
+Running all tests from the `logging` group, independent of the parent test groups can be done like this:
+
+```console
+tricot -v example.yml --groups **,logging
+```
+
+In addition to wildcards, *tricot* also support *brace expressions*. These can be used to constructed *or-like* test
+groups. E.g. to run the `logging` module from the `io` and `networking` parent test groups, you could use:
+
+```console
+tricot -v example.yml --groups {io,networking},logging
+```
+
+Wildcards and *brace expressions* can also be used together within a group specifications. Whereas *brace expressions* can
+be placed at any location of a group specification, wildcards are not allowed within the last comma separated value. Also
+for group matching, the `--exclude-groups` option triggers before the `--groups` option.
+
+Both, *IDs* and *test groups* are case sensitive.
+
+
### Environment Variables
----
diff --git a/resources/bash_completion.d/tricot b/resources/bash_completion.d/tricot
index fdf9ab2..b616992 100644
--- a/resources/bash_completion.d/tricot
+++ b/resources/bash_completion.d/tricot
@@ -12,7 +12,7 @@ function _tricot() {
COMPREPLY=()
# No completion
- if _comp_contains "--exclude --numbers --testers --positionals --variables" $prev; then
+ if _comp_contains "--exclude-ids --exclude-groups --groups --ids --positionals --variables" $prev; then
return 0
# Complete template modes
@@ -28,14 +28,15 @@ function _tricot() {
elif [[ "$cur" == -* ]]; then
opts="--help"
opts="$opts --debug"
- opts="$opts --exclude"
+ opts="$opts --exclude-ids"
+ opts="$opts --exclude-groups"
+ opts="$opts --groups"
+ opts="$opts --ids"
opts="$opts --logfile"
opts="$opts --load"
- opts="$opts --numbers"
opts="$opts --positionals"
opts="$opts --quite"
opts="$opts --template"
- opts="$opts --testers"
opts="$opts --variables"
opts="$opts --verbose"
diff --git a/setup.py b/setup.py
index d91cb95..17e83c0 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@
url='https://github.com/qtc-de/tricot',
name='tricot',
author='Tobias Neitzel (@qtc_de)',
- version='1.5.0',
+ version='1.6.0',
author_email='',
description='Trivial Command Testser',
diff --git a/tests/pytest/Misc/group_matching_test.py b/tests/pytest/Misc/group_matching_test.py
new file mode 100644
index 0000000..21ab50a
--- /dev/null
+++ b/tests/pytest/Misc/group_matching_test.py
@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+
+import tricot
+import pytest
+
+
+config_list = [['this,is,a,simple,group,def']]
+config_list.append(['now,lets,try,{conditional,orlike},group,def'])
+config_list.append(['{this,it},works,also,{with,using},more,than,one'])
+config_list.append(['multiple,lists', 'can,also,be,defined'])
+config_list.append(['{this,it},works', 'also,{with,using},orlike'])
+config_list.append(['lets,add,*,some,wildcards'])
+config_list.append(['lets,add,**,some,wildcards'])
+config_list.append(['lets,*,**,some,wildcards'])
+config_list.append(['**,some,wildcards'])
+
+match_list = [[([['this', 'is', 'a', 'simple', 'group', 'def']], True), ([['nope']], False)]]
+match_list.append([([['now', 'lets', 'try', 'conditional', 'group', 'def']], True),
+ ([['now', 'lets', 'try', 'orlike', 'group', 'def']], True),
+ ([['now', 'lets', 'tri', 'orlike', 'group', 'def']], False)])
+match_list.append([([['this', 'works', 'also', 'with', 'more', 'than', 'one']], True),
+ ([['this', 'works', 'also', 'using', 'more', 'than', 'one']], True),
+ ([['it', 'works', 'also', 'with', 'more', 'than', 'one']], True),
+ ([['it', 'works', 'also', 'using', 'more', 'than', 'one']], True),
+ ([['ot', 'works', 'also', 'using', 'more', 'than', 'one']], False)])
+match_list.append([([['aaaa', 'bbbb', 'cccc'], ['dddd', 'eeee']], False),
+ ([['aaaa', 'bbbb', 'cccc'], ['multiple', 'lists']], True),
+ ([['can', 'also', 'be', 'defined'], ['aaaa', 'bbbb']], True)])
+match_list.append([([['aaaa', 'bbbb', 'cccc'], ['dddd', 'eeee']], False),
+ ([['this', 'works'], ['multiple', 'lists']], True),
+ ([['also', 'using', 'orlike'], ['aaaa', 'bbbb']], True)])
+match_list.append([([['lets', 'add', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+ ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], False)])
+match_list.append([([['lets', 'add', 'hi :)', 'hi :D', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+ ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], False)])
+match_list.append([([['lets', 'add', 'hi :)', 'hi :D', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+ ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True)])
+match_list.append([([['lets', 'add', 'hi :)', 'hi :D', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+ ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+ ([['aaaaa', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True)])
+
+
+@pytest.mark.parametrize('config, match', zip(config_list, match_list))
+def test_group_matching(config, match):
+ '''
+ Check whether group matches are matching as expected
+ '''
+ for tupl in match:
+
+ groups = tricot.utils.parse_groups(config)
+ assert tricot.utils.groups_contain(groups, tupl[0]) == tupl[1]
diff --git a/tests/pytest/Misc/group_parsing_test.py b/tests/pytest/Misc/group_parsing_test.py
new file mode 100644
index 0000000..45539cd
--- /dev/null
+++ b/tests/pytest/Misc/group_parsing_test.py
@@ -0,0 +1,35 @@
+#!/usr/bin/python3
+
+import tricot
+import pytest
+
+
+config_list = [['this,is,a,simple,group,def']]
+config_list.append(['now,lets,try,{conditional,orlike},group,def'])
+config_list.append(['{this,it},works,also,{with,using},more,than,one'])
+config_list.append(['multiple,lists', 'can,also,be,defined'])
+config_list.append(['{this,it},works', 'also,{with,using},orlike'])
+
+result_list = [[['this', 'is', 'a', 'simple', 'group', 'def']]]
+result_list.append([['now', 'lets', 'try', 'conditional', 'group', 'def'],
+ ['now', 'lets', 'try', 'orlike', 'group', 'def']])
+result_list.append([['this', 'works', 'also', 'with', 'more', 'than', 'one'],
+ ['this', 'works', 'also', 'using', 'more', 'than', 'one'],
+ ['it', 'works', 'also', 'with', 'more', 'than', 'one'],
+ ['it', 'works', 'also', 'using', 'more', 'than', 'one']])
+result_list.append([['multiple', 'lists'], ['can', 'also', 'be', 'defined']])
+result_list.append([['this', 'works'], ['also', 'with', 'orlike'],
+ ['this', 'works'], ['also', 'using', 'orlike'],
+ ['it', 'works'], ['also', 'with', 'orlike'],
+ ['it', 'works'], ['also', 'using', 'orlike']])
+
+
+@pytest.mark.parametrize('config, results', zip(config_list, result_list))
+def test_group_parsing(config, results):
+ '''
+ Check whether group specifications are parsed correctly.
+ '''
+ groups = tricot.utils.parse_groups(config)
+
+ for group in groups:
+ assert group in results
diff --git a/tests/tricot/test-cases/extra/Groups.yml b/tests/tricot/test-cases/extra/Groups.yml
new file mode 100644
index 0000000..fe9dfd5
--- /dev/null
+++ b/tests/tricot/test-cases/extra/Groups.yml
@@ -0,0 +1,9 @@
+tester:
+ name: groups
+ title: Group Tests
+ description: |-
+ "Performs different checks on tricot's test group feature"
+
+testers:
+ - Groups/nested_one.yml
+ - Groups/nested_two.yml
diff --git a/tests/tricot/test-cases/extra/Groups/nested_four.yml b/tests/tricot/test-cases/extra/Groups/nested_four.yml
new file mode 100644
index 0000000..2387d2f
--- /dev/null
+++ b/tests/tricot/test-cases/extra/Groups/nested_four.yml
@@ -0,0 +1,36 @@
+tester:
+ name: group_nested_four
+ title: Nested Group Test Four
+ description: |-
+ "Performs different checks on tricot's test group feature"
+
+ groups:
+ - group_four
+
+tests:
+ - title: Test Group Four - One
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group four'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group Four - Two
+ groups:
+ - two
+ description: >
+ 'First test in test group four'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
diff --git a/tests/tricot/test-cases/extra/Groups/nested_one.yml b/tests/tricot/test-cases/extra/Groups/nested_one.yml
new file mode 100644
index 0000000..611903f
--- /dev/null
+++ b/tests/tricot/test-cases/extra/Groups/nested_one.yml
@@ -0,0 +1,40 @@
+tester:
+ name: group_nested_one
+ title: Nested Group Test One
+ description: |-
+ "Performs different checks on tricot's test group feature"
+
+ groups:
+ - group_one
+ - merge
+
+tests:
+ - title: Test Group One - One
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group one'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group One - Two
+ groups:
+ - two
+ description: >
+ 'First test in test group one'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+testers:
+ - ./nested_three.yml
diff --git a/tests/tricot/test-cases/extra/Groups/nested_three.yml b/tests/tricot/test-cases/extra/Groups/nested_three.yml
new file mode 100644
index 0000000..4b1e048
--- /dev/null
+++ b/tests/tricot/test-cases/extra/Groups/nested_three.yml
@@ -0,0 +1,36 @@
+tester:
+ name: group_nested_three
+ title: Nested Group Test Three
+ description: |-
+ "Performs different checks on tricot's test group feature"
+
+ groups:
+ - group_three
+
+tests:
+ - title: Test Group Three - One
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group three'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group Three - Two
+ groups:
+ - two
+ description: >
+ 'First test in test group three'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
diff --git a/tests/tricot/test-cases/extra/Groups/nested_two.yml b/tests/tricot/test-cases/extra/Groups/nested_two.yml
new file mode 100644
index 0000000..1c3143d
--- /dev/null
+++ b/tests/tricot/test-cases/extra/Groups/nested_two.yml
@@ -0,0 +1,40 @@
+tester:
+ name: group_nested_two
+ title: Nested Group Test Two
+ description: |-
+ "Performs different checks on tricot's test group feature"
+
+ groups:
+ - group_two
+ - merge
+
+tests:
+ - title: Test Group Two - One
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group two'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group Two - Two
+ groups:
+ - two
+ description: >
+ 'First test in test group two'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+testers:
+ - ./nested_four.yml
diff --git a/tests/tricot/test-cases/extra/IDs.yml b/tests/tricot/test-cases/extra/IDs.yml
new file mode 100644
index 0000000..8b1dc18
--- /dev/null
+++ b/tests/tricot/test-cases/extra/IDs.yml
@@ -0,0 +1,9 @@
+tester:
+ name: ids
+ title: ID Tests
+ description: |-
+ "Performs different checks on tricot's ID feature"
+
+testers:
+ - IDs/nested_one.yml
+ - IDs/nested_two.yml
diff --git a/tests/tricot/test-cases/extra/IDs/nested_four.yml b/tests/tricot/test-cases/extra/IDs/nested_four.yml
new file mode 100644
index 0000000..c2c1f09
--- /dev/null
+++ b/tests/tricot/test-cases/extra/IDs/nested_four.yml
@@ -0,0 +1,39 @@
+tester:
+ name: id_nested_four
+ title: Nested ID Test Four
+ description: |-
+ "Performs different checks on tricot's ID feature"
+
+ id: group_four
+ groups:
+ - group_four
+
+tests:
+ - title: Test Group Four - One
+ id: group_four_one
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group four'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group Four - Two
+ id: group_four_two
+ groups:
+ - two
+ description: >
+ 'Second test in test group four'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
diff --git a/tests/tricot/test-cases/extra/IDs/nested_one.yml b/tests/tricot/test-cases/extra/IDs/nested_one.yml
new file mode 100644
index 0000000..c456d53
--- /dev/null
+++ b/tests/tricot/test-cases/extra/IDs/nested_one.yml
@@ -0,0 +1,42 @@
+tester:
+ name: id_nested_one
+ title: Nested ID Test One
+ description: |-
+ "Performs different checks on tricot's id feature"
+
+ id: group_one
+ groups:
+ - group_one
+
+tests:
+ - title: Test Group One - One
+ id: group_one_one
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group one'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group One - Two
+ id: group_one_two
+ groups:
+ - two
+ description: >
+ 'Second test in test group one'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+testers:
+ - ./nested_three.yml
diff --git a/tests/tricot/test-cases/extra/IDs/nested_three.yml b/tests/tricot/test-cases/extra/IDs/nested_three.yml
new file mode 100644
index 0000000..eae503b
--- /dev/null
+++ b/tests/tricot/test-cases/extra/IDs/nested_three.yml
@@ -0,0 +1,39 @@
+tester:
+ name: id_nested_three
+ title: Nested ID Test Three
+ description: |-
+ "Performs different checks on tricot's ID feature"
+
+ id: group_three
+ groups:
+ - group_three
+
+tests:
+ - title: Test Group Three - One
+ id: group_three_one
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group three'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group Three - Two
+ id: group_three_two
+ groups:
+ - two
+ description: >
+ 'Second test in test group three'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
diff --git a/tests/tricot/test-cases/extra/IDs/nested_two.yml b/tests/tricot/test-cases/extra/IDs/nested_two.yml
new file mode 100644
index 0000000..3b24716
--- /dev/null
+++ b/tests/tricot/test-cases/extra/IDs/nested_two.yml
@@ -0,0 +1,42 @@
+tester:
+ name: id_nested_two
+ title: Nested ID Test Two
+ description: |-
+ "Performs different checks on tricot's id feature"
+
+ id: group_two
+ groups:
+ - group_two
+
+tests:
+ - title: Test Group Two - One
+ id: group_two_one
+ groups:
+ - one
+ - first
+ description: >
+ 'First test in test group two'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+ - title: Test Group Two - Two
+ id: group_two_two
+ groups:
+ - two
+ description: >
+ 'Second test in test group two'
+ command:
+ - echo
+ - 'Hi :D'
+ validators:
+ - contains:
+ values:
+ - 'Hi :D'
+
+testers:
+ - ./nested_four.yml
diff --git a/tests/tricot/tricot-tests/extra-tests.yml b/tests/tricot/tricot-tests/extra-tests.yml
index 64c8346..db1219c 100644
--- a/tests/tricot/tricot-tests/extra-tests.yml
+++ b/tests/tricot/tricot-tests/extra-tests.yml
@@ -5,6 +5,9 @@ tester:
'Perfors some additional tests on tricot, not related to plugins
or validators.'
+ id: 04
+ groups:
+ - extra
tests:
- title: Docker
@@ -272,3 +275,555 @@ tests:
invert:
- Fail... success
- Success... failed
+
+
+ - title: Groups All
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Groups One
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - group_one
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ invert:
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Groups Two
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - group_two
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+
+
+ - title: Groups One - One
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - group_one,one
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ invert:
+ - Test Group One - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Groups One / Two - One
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - '{group_one,group_two},one'
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group Two - One... success
+ invert:
+ - Test Group One - Two... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Groups One / Two - One (explicit)
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - 'group_one,one'
+ - 'group_two,one'
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group Two - One... success
+ invert:
+ - Test Group One - Two... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Groups One / Two * - Two
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - '{group_one,group_two},*,two'
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Three - Two... success
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Four - One... success
+
+
+ - title: Groups ** - One
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - '**,one'
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group Two - One... success
+ - Test Group Three - One... success
+ - Test Group Four - One... success
+ invert:
+ - Test Group One - Two... success
+ - Test Group Two - Two... success
+ - Test Group Three - Two... success
+ - Test Group Four - Two... success
+
+
+ - title: Exclude group_one
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --exclude-groups
+ - group_one
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+
+
+ - title: Exclude ** - One
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --exclude-groups
+ - '**,one'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - Two... success
+ - Test Group Two - Two... success
+ - Test Group Three - Two... success
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group Two - One... success
+ - Test Group Three - One... success
+ - Test Group Four - One... success
+
+
+ - title: Exclude vs Include
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --exclude-groups
+ - 'group_one'
+ - --groups
+ - 'group_one'
+
+ validators:
+ - error: False
+ - contains:
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Exclude vs Include 2
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --exclude-groups
+ - 'group_one'
+ - --groups
+ - '**,one'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Two - One... success
+ - Test Group Four - One... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Two - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Four - Two... success
+
+
+ - title: Exclude vs Include 3
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --exclude-groups
+ - 'group_one,one'
+ - --groups
+ - 'group_one'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Group Merge
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --groups
+ - 'merge'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: Group Merge Exclude
+ description: |-
+ "Test tricot's test group feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/Groups.yml
+ - --exclude-groups
+ - 'group_one'
+ - --groups
+ - 'merge'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+
+
+ - title: ID Test - One Two
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --ids
+ - 'group_one'
+ - 'group_two'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+
+
+ - title: ID Test - One
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --ids
+ - 'group_one'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ invert:
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+ - Test Group Four - Two... success
+
+
+ - title: ID Test - FourOne
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --ids
+ - 'group_four_one'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Four - One... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - Two... success
+
+
+ - title: Exclude ID
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --exclude-ids
+ - 'group_four_one'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - Two... success
+ invert:
+ - Test Group Four - One... success
+
+
+ - title: Exclude vs Include (ID)
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --exclude-ids
+ - 'group_four_one'
+ - --ids
+ - 'group_four_one'
+
+ validators:
+ - error: False
+ - contains:
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - Two... success
+ - Test Group Four - One... success
+
+
+ - title: Exclude vs Include 2 (ID)
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --exclude-ids
+ - 'group_four_one'
+ - --ids
+ - 'group_four'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
+
+
+ - title: Exclude vs Include (Groups vs ID)
+ description: |-
+ "Test tricot's ID feature"
+
+ command:
+ - tricot
+ - ${EXTRA}/IDs.yml
+ - --exclude-ids
+ - 'group_four_one'
+ - --groups
+ - 'group_two,group_four'
+
+ validators:
+ - error: False
+ - contains:
+ values:
+ - Test Group Four - Two... success
+ invert:
+ - Test Group One - One... success
+ - Test Group One - Two... success
+ - Test Group Three - One... success
+ - Test Group Three - Two... success
+ - Test Group Two - One... success
+ - Test Group Two - Two... success
+ - Test Group Four - One... success
diff --git a/tests/tricot/tricot-tests/extractor-tests.yml b/tests/tricot/tricot-tests/extractor-tests.yml
index 0c21d4b..2359f5f 100644
--- a/tests/tricot/tricot-tests/extractor-tests.yml
+++ b/tests/tricot/tricot-tests/extractor-tests.yml
@@ -4,6 +4,10 @@ tester:
description: |-
"Perform tests on all tricot extractors"
+ id: 03
+ groups:
+ - extractor
+
tests:
- title: RegexExtractor
diff --git a/tests/tricot/tricot-tests/plugin-tests.yml b/tests/tricot/tricot-tests/plugin-tests.yml
index 23ca3b7..d694f30 100644
--- a/tests/tricot/tricot-tests/plugin-tests.yml
+++ b/tests/tricot/tricot-tests/plugin-tests.yml
@@ -4,6 +4,9 @@ tester:
description: |-
"Performs tests on all tricot plugins"
+ id: 02
+ groups:
+ - plugin
tests:
- title: CleanupPlugin
diff --git a/tests/tricot/tricot-tests/validator-tests.yml b/tests/tricot/tricot-tests/validator-tests.yml
index de7e458..fed5557 100644
--- a/tests/tricot/tricot-tests/validator-tests.yml
+++ b/tests/tricot/tricot-tests/validator-tests.yml
@@ -4,6 +4,9 @@ tester:
description: |-
"Performs tests on all tricot validators"
+ id: 01
+ groups:
+ - validator
tests:
- title: ContainsValidator
diff --git a/tricot/constants.py b/tricot/constants.py
index 2ed4d0d..9c68b71 100644
--- a/tricot/constants.py
+++ b/tricot/constants.py
@@ -17,5 +17,8 @@
CONDITION_FORMAT_ERROR = 16
EXTRACT_EXCEPTION = 17
EXTRACTOR_ERROR = 18
+DUPLICATE_ID_ERROR = 19
+YAML_SYNTAX_ERROR = 20
LAST_ERROR = 0
+VERSION = '1.6.0'
diff --git a/tricot/logging.py b/tricot/logging.py
index 4c741d6..79d43e6 100644
--- a/tricot/logging.py
+++ b/tricot/logging.py
@@ -59,6 +59,12 @@ def print_plain_red(string: str, end: str = None) -> None:
'''
cprint(string, color='red', end=end)
+ def print_plain_blue(string: str, end: str = None) -> None:
+ '''
+ Print without indent or prefix, text in blue.
+ '''
+ cprint(string, color='blue', end=end)
+
def print_yellow(string: str, e: bool = False, end: str = None) -> None:
'''
Print with prefix and indent, text in yellow.
diff --git a/tricot/tricot.py b/tricot/tricot.py
index cf7a30d..a1d6daf 100644
--- a/tricot/tricot.py
+++ b/tricot/tricot.py
@@ -16,6 +16,15 @@
from tricot.condition import Condition
+assigned_ids = set()
+
+
+class DuplicateIDError(Exception):
+ '''
+ Custom exception that is raised if two testers/tests use the same ID.
+ '''
+
+
class TricotException(Exception):
'''
Custom exception class for general purpose and tricot related exceptions.
@@ -107,11 +116,11 @@ class Test:
evaluated by executing their 'run' method.
'''
expected_keys = ['title', 'description', 'command', 'arguments', 'validators', 'timeout', 'env', 'conditions',
- 'logfile', 'shell', 'extractors']
+ 'logfile', 'shell', 'extractors', 'id', 'groups']
def __init__(self, path: Path, title: str, error_mode: str, variables: dict[str, Any], command: Command,
timeout: int, validators: list[Validator], extractors: list[Extractor], env: dict, conditions: dict,
- conditionals: set[Condition]) -> None:
+ conditionals: set[Condition], test_id: str, test_groups: list[list[str]]) -> None:
'''
Initializer for a Test object.
@@ -127,6 +136,8 @@ def __init__(self, path: Path, title: str, error_mode: str, variables: dict[str,
env Environment variables
conditions Conditions for running the current test
conditionals Conditionals defined by upper testers
+ test_id Identifikation number of the test
+ test_groups Test groups that the test belongs to
Returns:
None
@@ -143,6 +154,19 @@ def __init__(self, path: Path, title: str, error_mode: str, variables: dict[str,
self.conditions = conditions
self.conditionals = conditionals
+ if test_id is None:
+ self.id = self.title
+
+ else:
+ self.id = str(test_id)
+
+ if self.id in assigned_ids:
+ raise DuplicateIDError(f"ID '{self.id}' was assigned twice.")
+
+ assigned_ids.add(self.id)
+
+ self.groups = test_groups
+
self.logfile = None
self.success_string = 'success'
self.failure_string = 'failed'
@@ -236,7 +260,8 @@ def apply_variables(val: Union(str, list), variables: dict[str, Any], k: str = '
return val
def from_dict(path: Path, input_dict: dict, variables: dict[str, Any] = {}, error_mode: str = 'continue',
- environment: dict = {}, conditionals: set[Condition] = set(), output_conf: dict = {}) -> Test:
+ environment: dict = {}, conditionals: set[Condition] = set(), output_conf: dict = {},
+ parent_groups: list[list[str]] = list()) -> Test:
'''
Creates a Test object from a dictionary. The dictionary is expected to be the content
read in of a .yml file and needs all keys that are required for a test (validators,
@@ -250,6 +275,7 @@ def from_dict(path: Path, input_dict: dict, variables: dict[str, Any] = {}, erro
environment Environment variables
conditionals Conditionals for running the test
output_conf Output configuration inherited by the tester
+ parent_groups Test groups inherited from the parent tester
Returns:
Test Newly generated Test object
@@ -259,6 +285,7 @@ def from_dict(path: Path, input_dict: dict, variables: dict[str, Any] = {}, erro
var = tricot.utils.merge(variables, j.get('variables', {}), 'variables', path)
validators = Validator.from_list(path, j['validators'], var)
extractors = Extractor.from_list(path, j.get('extractors', []), var)
+ groups = tricot.utils.merge_groups(parent_groups, list(map(lambda x: str(x), j.get('groups', []))))
e_mode = j.get('error_mode') or error_mode
env = tricot.utils.merge_environment(j.get('env'), environment, path)
@@ -284,7 +311,7 @@ def from_dict(path: Path, input_dict: dict, variables: dict[str, Any] = {}, erro
tricot.utils.check_keys(Test.expected_keys, input_dict)
test = Test(path, j['title'], e_mode, var, command, j.get('timeout'), validators, extractors, env,
- conditions, conditionals)
+ conditions, conditionals, j.get('id'), groups)
test.set_logfile(j.get('logfile'))
test.set_output(tricot.utils.merge(output_conf, j.get('output', {}), 'output', path))
@@ -298,7 +325,8 @@ def from_dict(path: Path, input_dict: dict, variables: dict[str, Any] = {}, erro
raise TestKeyError(None, path, str(e))
def from_list(path: Path, input_list: list, variables: dict[str, Any] = {}, error_mode: str = 'continue',
- env: dict = {}, conditionals: set[Condition] = set(), output_conf: dict = {}) -> list[Test]:
+ env: dict = {}, conditionals: set[Condition] = set(), output_conf: dict = {},
+ parent_groups: list[list[str]] = list()) -> list[Test]:
'''
Within .yml files, Tests are specified in form of a list. This function takes such a list,
that contains each single test definition as another dictionary (like it is created when
@@ -312,6 +340,7 @@ def from_list(path: Path, input_list: list, variables: dict[str, Any] = {}, erro
env Environment variables
conditionals Conditionals specified by the upper tester
output_conf Output configuration inherited by the tester
+ parent_groups Test groups inherited from the parent tester
Returns
list[Test] List of Test objects created from the .yml input
@@ -324,7 +353,8 @@ def from_list(path: Path, input_list: list, variables: dict[str, Any] = {}, erro
for ctr in range(len(input_list)):
try:
- test = Test.from_dict(path, input_list[ctr], variables, error_mode, env, conditionals, output_conf)
+ test = Test.from_dict(path, input_list[ctr], variables, error_mode, env, conditionals,
+ output_conf, parent_groups)
tests.append(test)
except TestKeyError as e:
@@ -348,7 +378,12 @@ def run(self, prefix: str = '-', hotplug_variables: dict[str, Any] = None) -> No
None
'''
Logger.add_logfile(self.logfile)
- Logger.print_blue(f'{prefix} {self.title}...', end=' ', flush=True)
+
+ if self.id and self.id != self.title:
+ Logger.print_blue(f'{prefix} [{self.id}] {self.title}...', end=' ', flush=True)
+ else:
+ Logger.print_blue(f'{prefix} {self.title}...', end=' ', flush=True)
+
success = True
if not Condition.check_conditions(self.conditions, self.conditionals):
@@ -421,6 +456,25 @@ def run(self, prefix: str = '-', hotplug_variables: dict[str, Any] = None) -> No
hotplug_variables['$prev'] = self.command
Logger.remove_logfile(self.logfile)
+ def skip_test(self, exclude: set[str], exclude_groups: list[list[str]]) -> bool:
+ '''
+ Checks whether the current test is contained within the exclude lists.
+
+ Parameters:
+ exclude Set of Test / Tester IDs to exclude
+ exclude_groups List of group sets to exclude
+
+ Returns:
+ bool
+ '''
+ if exclude and self.id in exclude:
+ return True
+
+ elif exclude_groups and tricot.utils.groups_contain(exclude_groups, self.groups):
+ return True
+
+ return False
+
class Tester:
'''
@@ -433,7 +487,7 @@ class Tester:
def __init__(self, path: Path, name: str, title: str, variables: dict[str, Any], tests: list[Test], testers: list[Tester],
containers: list[TricotContainer], plugins: list[Plugin], conditions: dict, conditionals: set[Condition],
- error_mode: str) -> None:
+ error_mode: str, tester_id: str, test_groups: list[list[str]]) -> None:
'''
Initializes a new Tester object.
@@ -449,6 +503,11 @@ def __init__(self, path: Path, name: str, title: str, variables: dict[str, Any],
conditions Conditions for running the current tester
conditionals Conditionals defined by the tester or upper testers
error_mode Decides what to do if a plugin fails (break|continue)
+ tester_id Unique identifikation number of the tester
+ test_groups Test groups that the tester belongs to
+
+ Returns:
+ None
'''
self.name = name
self.title = title or name
@@ -461,11 +520,26 @@ def __init__(self, path: Path, name: str, title: str, variables: dict[str, Any],
self.conditionals = conditionals
self.error_mode = error_mode
+ if tester_id is None:
+ self.id = self.name
+
+ else:
+ self.id = str(tester_id)
+
+ if self.id in assigned_ids:
+ raise DuplicateIDError(f"ID '{self.id}' was assigned twice.")
+
+ assigned_ids.add(self.id)
+
+ self.groups = test_groups
+
self.logfile = None
+ self.runall = False
def from_dict(input_dict: dict, initial_vars: dict[str, Any] = dict(),
path: Path = None, e_mode: str = None, environment: dict = {},
- conditionals: set[Condition] = set(), output_conf: dict = {}) -> Tester:
+ conditionals: set[Condition] = set(), output_conf: dict = {},
+ test_groups: list[list[str]] = []) -> Tester:
'''
Creates a new Tester object from a python dictionary. The dictionary is expected to be
created by reading a .yml file that contains test defintions. It requires all keys that
@@ -480,6 +554,7 @@ def from_dict(input_dict: dict, initial_vars: dict[str, Any] = dict(),
environment Dictionary of environment variables to use within the test
conditionals Conditions inherited from the upper tester
output_conf Output configuration inherited from the upper tester
+ test_groups List of test groups inherited from the upper tester
Returns:
Tester Tester object created from the dictionary
@@ -498,6 +573,7 @@ def from_dict(input_dict: dict, initial_vars: dict[str, Any] = dict(),
testers = g.get('testers')
definitions = g.get('tests')
+ groups = tricot.utils.merge_groups(test_groups, list(map(lambda x: str(x), t.get('groups', []))))
variables = tricot.utils.merge(initial_vars, g.get('variables', {}), 'variables', path)
variables['cwd'] = path.parent
@@ -515,18 +591,18 @@ def from_dict(input_dict: dict, initial_vars: dict[str, Any] = dict(),
f = path.parent.joinpath(f)
for ff in glob.glob(str(f)):
- tester = Tester.from_file(ff, variables, None, error_mode, env, conds, output_c)
+ tester = Tester.from_file(ff, variables, None, error_mode, env, conds, output_c, groups)
tester_list.append(tester)
tests = None
if definitions and type(definitions) is list:
- tests = Test.from_list(path, definitions, variables, error_mode, env, conds, output_c)
+ tests = Test.from_list(path, definitions, variables, error_mode, env, conds, output_c, groups)
elif not tester_list:
raise TesterKeyError('tests', path, optional='testers')
new_tester = Tester(path, t['name'], t.get('title'), variables, tests, tester_list, containers, plugins,
- run_conds, conds, error_mode)
+ run_conds, conds, error_mode, t.get('id'), groups)
new_tester.set_logfile(t.get('logfile'))
return new_tester
@@ -552,7 +628,7 @@ def set_logfile(self, logfile: str) -> None:
def from_file(filename: str, initial_vars: dict[str, Any] = dict(), runtime_vars: dict[str, Any] = None,
error_mode: str = None, env: dict = {}, conditionals: set[Condition] = set(),
- output_conf: dict = {}) -> Tester:
+ output_conf: dict = {}, test_groups: list[list[str]] = []) -> Tester:
'''
Creates a new Tester object from a .yml file. The .yml file obviously needs to be in the
expected format and requires all keys that are needed to construct a Tester object.
@@ -565,6 +641,7 @@ def from_file(filename: str, initial_vars: dict[str, Any] = dict(), runtime_vars
env Current environment variables
conditionals Conditions inherited from the previous tester
output_conf Output configuration inherited from upper tester
+ test_groups List of test groups inherited from the upper tester
Returns:
Tester Tester object created from the file
@@ -578,61 +655,142 @@ def from_file(filename: str, initial_vars: dict[str, Any] = dict(), runtime_vars
if runtime_vars is not None and '$runtime' not in initial_vars:
initial_vars['$runtime'] = runtime_vars
- return Tester.from_dict(config_dict, initial_vars, Path(filename), error_mode, env, conditionals, output_conf)
+ return Tester.from_dict(config_dict, initial_vars, Path(filename), error_mode, env, conditionals,
+ output_conf, test_groups)
+
+ def set_runall(self, value: bool) -> None:
+ '''
+ Sets the runall property of a tester. This is required when users specify a tetser ID on the command
+ line. In this case, all tests and nested testers should be run, although their ID is not contained
+ within the IDs to run. Setting the runall property on a tester disabled ID checking and runs everything
+ inside of it anyway.
+
+ Setting the runall property is done recursively for all nested testers.
+
+ Parameters:
+ value Value to set for the runall property.
+
+ Returns:
+ None
+ '''
+ self.runall = value
- def contains_testers(self, testers: list[str]) -> bool:
+ for tester in self.testers:
+ tester.set_runall(value)
+
+ def contains_id(self, t_ids: set[str]) -> bool:
'''
- Checks whether the specified tester name is contained within the current test tree.
+ Checks whether the specified Test / Tester IDs are contained within this tester.
Parameters:
- testers List of tester names to look for
+ t_ids Test / Tester IDs to look for
Returns:
- bool True if tester is contained within test tree
+ bool True if one of the ids is contained within the tester
'''
- if len(testers) == 0:
+ if not t_ids or self.runall:
return True
- for tester in testers:
+ if self.id and {self.id}.issubset(t_ids):
+ self.set_runall(True)
+ return True
- if self.name == tester:
- return True
+ if self.tests:
+
+ for test in self.tests:
+ if test.id and {test.id}.issubset(t_ids):
+ return True
for tester in self.testers:
+ if tester.contains_id(t_ids):
+ return True
+
+ return False
+
+ def contains_group(self, t_groups: list[list[str]]) -> bool:
+ '''
+ Checks whether the specified Group set exists within the tester.
- if tester.contains_testers(testers):
+ Parameters:
+ t_groups List of group lists to check in
+
+ Returns:
+ bool True if set of groups is contained within the tester
+ '''
+ if not t_groups or self.runall:
+ return True
+
+ if tricot.utils.groups_contain(t_groups, self.groups):
+ self.set_runall(True)
+ return True
+
+ if self.tests:
+
+ for test in self.tests:
+ if tricot.utils.groups_contain(t_groups, test.groups):
+ return True
+
+ for tester in self.testers:
+ if tester.contains_group(t_groups):
return True
return False
- def run(self, testers: list[str] = (), numbers: list[str] = (), exclude: list[str] = (),
- hotplug_variables: dict[str, Any] = {}) -> None:
+ def skip_test(self, exclude: set[str], exclude_groups: list[list[str]]) -> bool:
+ '''
+ Checks whether the current test is contained within the exclude lists.
+
+ Parameters:
+ exclude Set of Test / Tester IDs to exclude
+ exclude_groups List of group lists to exclude
+
+ Returns:
+ bool
+ '''
+ if exclude and self.id in exclude:
+ return True
+
+ elif exclude_groups and tricot.utils.groups_contain(exclude_groups, self.groups):
+ return True
+
+ return False
+
+ def run(self, ids: set[str], groups: list[list[str]], exclude: set[str],
+ exclude_groups: list[list[str]], hotplug_variables: dict[str, Any] = dict()) -> None:
'''
Runs the test: Prints the title of the tester and iterates over all contained
Test objects and calls their 'run' method.
Parameters:
- tester Only run testers that match the specified names
- numbers Only run tests that match the specified numbers
- exclude Exclude the specified tester names
- hotplug_variables Dictionary of variables that are applied at runtime.
+ ids Set of Test / Tester IDs to run
+ groups List of group lists to run
+ exclude Set of Test / Tester IDs to exclude
+ exclude_groups List of group lists to exclude
+ hotplug_variables Hotplug variables to use during the test
Returns:
None
'''
- if not self.contains_testers(testers) or self.name in exclude:
+ if self.skip_test(exclude, exclude_groups):
return
- Tester.increase()
+ if not self.contains_id(ids) or not self.contains_group(groups):
+ return
if not Condition.check_conditions(self.conditions, self.conditionals):
Logger.print_mixed_yellow('Skipping test:', self.title)
return
+ tricot.Logger.print('')
Logger.add_logfile(self.logfile)
- Logger.print_mixed_yellow('Starting test:', self.title)
- hotplug = hotplug_variables.copy()
+ Logger.print_mixed_yellow('Starting test:', self.title, end=' ')
+
+ if self.id and self.id != self.name:
+ Logger.print_plain_blue(f'[{self.id}]')
+ else:
+ print()
+ hotplug = hotplug_variables.copy()
Logger.increase_indent()
for plugin in self.plugins:
@@ -642,13 +800,8 @@ def run(self, testers: list[str] = (), numbers: list[str] = (), exclude: list[st
container.start_container()
hotplug = {**hotplug, **container.get_container_variables()}
- if self.tests:
- Logger.print('')
- for ctr in range(len(self.tests)):
- if len(numbers) == 0 or (ctr+1) in numbers:
- self.tests[ctr].run(f'{ctr+1}.', hotplug)
-
- self.run_childs(testers, numbers, exclude, hotplug)
+ self.run_tests(ids, groups, exclude, exclude_groups, hotplug)
+ self.run_childs(ids, groups, exclude, exclude_groups, hotplug)
for container in self.containers:
container.stop_container()
@@ -659,28 +812,70 @@ def run(self, testers: list[str] = (), numbers: list[str] = (), exclude: list[st
Logger.remove_logfile(self.logfile)
Logger.decrease_indent()
- def run_childs(self, testers: list[str] = (), numbers: list[str] = (), exclude: list[str] = (),
- hotplug_variables: dict[str, Any] = {}) -> None:
+ def run_tests(self, ids: set[str], groups: list[list[str]], exclude: set[str],
+ exclude_groups: list[list[str]], hotplug_variables: dict[str, Any]) -> None:
+ '''
+ Wrapper function that executes the tests specified in a tester.
+
+ Parameters:
+ ids Set of Test / Tester IDs to run
+ groups List of group lists to run
+ exclude Set of Test / Tester IDs to exclude
+ exclude_groups List of group lists to exclude
+ hotplug_variables Hotplug variables to use during the test
+
+ Returns:
+ None
+ '''
+ if not self.tests:
+ return
+
+ Logger.print('')
+ runall = tricot.utils.groups_contain(groups, self.groups) or (ids and {self.id}.issubset(ids))
+
+ for ctr in range(len(self.tests)):
+
+ test = self.tests[ctr]
+
+ if test.skip_test(exclude, exclude_groups):
+ continue
+
+ elif self.runall or runall:
+ pass
+
+ elif not ids and not groups:
+ pass
+
+ elif ids and {test.id}.issubset(ids):
+ pass
+
+ elif groups and tricot.utils.groups_contain(groups, test.groups):
+ pass
+
+ else:
+ continue
+
+ test.run(f'{ctr+1}.', hotplug_variables)
+
+ def run_childs(self, ids: set[str], groups: list[list[str]], exclude: set[str],
+ exclude_groups: list[list[str]], hotplug_variables: dict[str, Any]) -> None:
'''
Runs the child testers of the current tester.
Parameters:
- tester Only run testers that match the specified names
- numbers Only run tests that match the specified numbers
- exclude Exclude the specified tester names
- hotplug_variables Dictionary of variables that are applied at runtime.
+ ids Set of Test / Tester IDs to run
+ groups List of group lists to run
+ exclude Set of Test / Tester IDs to exclude
+ exclude_groups List of group lists to exclude
+ hotplug_variables Hotplug variables to use during the test
Returns:
None
'''
for tester in self.testers:
- try:
- if self.name in testers:
- tester.run(numbers=numbers, exclude=exclude, hotplug_variables=hotplug_variables)
-
- else:
- tester.run(testers, numbers, exclude, hotplug_variables)
+ try:
+ tester.run(ids, groups, exclude, exclude_groups, hotplug_variables)
except tricot.PluginException as e:
diff --git a/tricot/utils.py b/tricot/utils.py
index 8514eb7..ba494cf 100644
--- a/tricot/utils.py
+++ b/tricot/utils.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import os
+import re
import tricot
from typing import Any
from pathlib import Path
@@ -275,3 +276,146 @@ def merge(dict1: dict, dict2: dict, name: str = None, path: Path = None) -> dict
raise tricot.TricotException('Invalid type specified during merge operation.', path)
return {**dict1, **dict2}
+
+
+def parse_groups(groups: list[str]) -> list[list[str]]:
+ '''
+ Parses group specifications. Groups should be specified as comma separated strings. Each
+ comma separated part is interpreted as a group. All groups within a string are mandatory
+ for a test / tester to match. Braces can be used for or-like statements.
+
+ E.g.:
+
+ java8,networking,filter -> list(java8, networking, filter)
+ java8,{networking,io},filter -> list(list(java8, networking, filter),
+ list(java8, io, filter))
+
+ Parameters:
+ groups List of group specifications
+
+ Returns:
+ list List of group lists
+ '''
+ lists = list()
+ regex = re.compile(r'\{([^}]+)\}')
+
+ for group_spec in groups:
+
+ or_like = regex.findall(group_spec)
+ group_spec = regex.sub('$ORLIKE$', group_spec)
+
+ split = list(filter(None, group_spec.split(',')))
+ group_lists = [split]
+
+ for match in or_like:
+
+ new = []
+
+ for group_list in group_lists:
+
+ split = list(filter(None, match.split(',')))
+
+ for item in split:
+
+ new_list = group_list.copy()
+
+ for ctr in range(len(new_list)):
+
+ if new_list[ctr] == '$ORLIKE$':
+ new_list[ctr] = item
+ break
+
+ new.append(new_list)
+
+ group_lists = new
+
+ lists += group_lists
+
+ return lists
+
+
+def groups_contain(groups_list: list[list[str]], groups: list[list[str]]) -> bool:
+ '''
+ Checks whether a specified list of groups contains a particular group of a
+ list of specified groups. This separate function is required, as group
+ comparison supports wildcards as * or **.
+
+ Parameters:
+ groups_list List of group lists to search in
+ groups Group list to look for
+
+ Returns:
+ bool True if group is contained in groups
+ '''
+ for group in groups:
+
+ for items in groups_list:
+
+ ctr = 0
+ match = True
+ items = items.copy()
+
+ try:
+
+ while len(items) != 0:
+
+ item = items.pop(0)
+
+ if item == '*' or group[ctr] == item:
+
+ ctr += 1
+ continue
+
+ if item == '**':
+
+ item = items.pop(0)
+ while group[ctr] != item and ctr != len(group):
+ ctr += 1
+
+ if ctr == len(group):
+ match = False
+ break
+
+ ctr += 1
+ continue
+
+ match = False
+ break
+
+ if match:
+ return True
+
+ except IndexError:
+ pass
+
+ return False
+
+
+def merge_groups(parent_groups: list[list[str]], new_groups: list[str]) -> list[list[str]]:
+ '''
+ This function is called by tests and testers to join groups that are defined within
+ the test / tester definition with group lists that have been specified for upper testers.
+ Each group in the test / tester specification is appened to the parent defined groups.
+
+ Paramaters:
+ parent_groups Group lists inherited by the parent
+ new_groups Groups specified for the test / tester
+
+ Returns:
+ merged Merge result
+ '''
+ merged = list()
+
+ for parent_group in (parent_groups or [[]]):
+
+ if new_groups:
+
+ for new_group in new_groups:
+ copy = parent_group.copy()
+ copy.append(new_group)
+ merged.append(copy)
+
+ else:
+ merged.append(parent_group.copy())
+
+ return merged