Merge pull request #14 from qtc-de/develop

Prepare v1.6.0 Release

CHANGELOG.md

@@ -6,6 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [1.6.0] - Oct 09, 2021
+
+### Added
+
+* Add test / tester *IDs* (see https://github.com/qtc-de/tricot/tree/main/docs#selective-testing)
+* Add *test groups* (see https://github.com/qtc-de/tricot/tree/main/docs#selective-testing)
+
+### Changed
+
+* Full help menu is now always displayed on argparse errors
+* Small bugfixes and formatting changes
+
+### Removed
+
+* The `--tester`, `--exclude` and `--number` options were removed.
+  They are replaced by the `--ids`, `--groups`, `--exclude-ids` and
+  `--exclude-groups` options.
+
+
 ## [1.5.0] - Sep 11, 2021
 
 ### Added

bin/tricot

@@ -9,22 +9,39 @@ import docker
 import tricot
 import argparse
 
+from tricot.constants import VERSION
 
-parser = argparse.ArgumentParser(description='''tricot v1.5.0 - a trivial command tester that allows you to verify that certain
-                                                commands or executables behave as expected. It uses .yml files for test
-                                                definitions and can be used from the command line or as a python library.''',
-                                                formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=40))
+
+class FullHelpParser(argparse.ArgumentParser):
+    '''
+    Custom ArgumentParser class to show more helpful help messages per default.
+    Taken from: https://stackoverflow.com/a/4042861
+    '''
+    def error(self, message):
+        '''
+        Show the whole help menu per default.
+        '''
+        sys.stderr.write(f'error: {message}\n\n')
+        self.print_help()
+        sys.exit(2)
+
+
+parser = FullHelpParser(description=f'''tricot v{VERSION} - a trivial command tester to verify that commands,
+                                        scripts or executables behave as expected. It uses .yml files for test
+                                        definitions and can be used from the command line or as a python library.''',
+                                        formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=60))
 
 parser.add_argument('--debug', dest='debug', action='store_true', help='enable debug output')
-parser.add_argument('--exclude', metavar='name', nargs='+', default=[], help='exclude the specified testers')
+parser.add_argument('--exclude-ids', dest='eids', metavar='id', nargs='+', default=[], help='exclude the specified test / tester IDs')
+parser.add_argument('--exclude-groups', dest='egroups', metavar='group', nargs='+', default=[], help='exclude the specified test groups')
 parser.add_argument('file', metavar='file', nargs='+', help='test definition (.yml file)')
-parser.add_argument('--logfile', dest='log', type=argparse.FileType('w'), help='mirror output into a logfile')
+parser.add_argument('--groups', metavar='group', nargs='+', default=[], help='only run the specified test groups')
+parser.add_argument('--ids', metavar='id', nargs='+', default=[], help='only run the specified test / tester IDs')
+parser.add_argument('--logfile', dest='log', metavar='file', type=argparse.FileType('w'), help='mirror output into a logfile')
 parser.add_argument('--load', dest='load', metavar='file', nargs='+', default=[], type=argparse.FileType('r'), help='custom validators, extractors and plugins')
-parser.add_argument('--numbers', dest='numbers', metavar='int', type=int, nargs='+', default=[], help='only run the specified test numbers')
-parser.add_argument('--positionals', dest='pos', metavar='pos', nargs='+', default=[], help='positional variables (accessible by $1, $2, ...)')
+parser.add_argument('--positionals', dest='pos', metavar='var', nargs='+', default=[], help='positional variables (accessible by $1, $2, ...)')
 parser.add_argument('-q', '--quite', dest='quite', action='store_true', help='disable verbose output during tests')
 parser.add_argument('--template', dest='template', choices=['tester', 'plugin', 'validator', 'extractor'], help='write a template file')
-parser.add_argument('--testers', dest='testers', metavar='name', nargs='+', default=[], help='only run the specified testers')
 parser.add_argument('--variables', dest='vars', metavar='vars', nargs='+', default=[], help='runtime variables')
 parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='enable verbose logging during tests')
 
@@ -152,11 +169,16 @@ def main():
     load(args.load)
     variables = prepare_variables(args)
 
+    groups = tricot.utils.parse_groups(args.groups)
+    egroups = tricot.utils.parse_groups(args.egroups)
+
+    tricot.Logger.print_mixed_yellow('tricot', f'v{VERSION}', '- Starting tests...')
+
     for yml_file in args.file:
 
         try:
             tester = tricot.Tester.from_file(yml_file, runtime_vars=variables)
-            tester.run(args.testers, args.numbers, args.exclude)
+            tester.run(set(args.ids), groups, set(args.eids), egroups)
 
         except tricot.ValidatorError as e:
             tricot.Logger.print_mixed_red('Caught', 'ValidatorError', 'while parsing test configuration.', e=True)
@@ -249,13 +271,25 @@ def main():
             tricot.Logger.print_with_indent_blue(str(e), e=True)
             sys.exit(tricot.constants.DOCKER_API)
 
+        except tricot.DuplicateIDError as e:
+            tricot.Logger.print_mixed_yellow('Caught', 'DuplicateIDError', 'while parsing test configuration.', e=True)
+            tricot.Logger.print_with_indent_blue(str(e), e=True)
+            sys.exit(tricot.constants.DUPLICATE_ID_ERROR)
+
         except tricot.TricotRuntimeError as e:
             tricot.Logger.print_mixed_blue('Caught', 'unexpected Exception', 'while running the test command.', e=True)
             tricot.Logger.print_yellow('Original Error:', e=True)
             tricot.Logger.increase_indent()
             tricot.Logger.print_with_indent_blue(str(e.original), e=True)
             sys.exit(tricot.constants.RUNTIME_ERROR)
 
+        except yaml.scanner.ScannerError as e:
+            tricot.Logger.print_mixed_yellow('Caught', 'yaml.scanner.ScannerError', 'while parsing test configuration.', e=True)
+            tricot.Logger.print('Seems that there is a syntax error within your test configuration.', e=True)
+            tricot.Logger.increase_indent()
+            tricot.Logger.print_with_indent_blue(str(e), e=True)
+            sys.exit(tricot.constants.YAML_SYNTAX_ERROR)
+
         except KeyboardInterrupt:
             tricot.Logger.reset_indent()
             tricot.Logger.print('')
@@ -267,7 +301,7 @@ def main():
             if args.debug:
                 raise e
 
-            tricot.Logger.print_mixed_yellow('Caught', 'unexpected Exception', e=True)
+            tricot.Logger.print_mixed_yellow('Caught', 'unexpected Exception', 'while running tricot.', e=True)
             tricot.Logger.increase_indent()
             tricot.Logger.print_with_indent_blue(str(e), e=True)
             sys.exit(tricot.constants.UNKNOWN)

docs/README.md

@@ -9,8 +9,9 @@ on it's own.
 - [Extractor, Validator and Plugin List](#extractor-validator-and-plugin-list)
 - [Writing Custom Plugins](#writing-custom-plugins)
 - [Writing Custom Validators](#writing-custom-validators)
-- [Writing Custom Extactors](#writing-custom-extractors)
+- [Writing Custom Extractors](#writing-custom-extractors)
 - [Accessing Command Information from an Validator](#accessing-command-information-from-an-validator)
+- [Selective Testing](#selective-testing)
 - [Environment Variables](#environment-variables)
 - [Runtime Variables](#runtime-variables)
 - [Nesting Variables](#nesting-variables)
@@ -354,7 +355,7 @@ plugins:
     key2: 22
 ```
 
-* ``param_type`` can be used to specify the python type that is expected for the toplevel argument of a
+* ``param_type`` can be used to specify the python type that is expected for the top level argument of a
   plugin or validator. In the example test configuration above, the ``param_type`` values should be set
   like this:
   * ``example_one``: ``param_type = str``
@@ -372,6 +373,108 @@ The parameter validation described above is very basic and has obviously limitat
 want a parameter validation that is easier to use and has an arbitrary recursion depth.
 
 
+### Selective Testing
+
+----
+
+A common testing scenario is that you just changed a portion of a program and only want to run tests for the affected
+component. *tricot* supports this *selective testing* approach by using *IDs* and *test groups*.
+
+*IDs* are exactly what the name suggests, a unique identifier for each test / tester. You can assign them by using the
+`id` key within of test definitions. *IDs* are ordinary strings and can contain any characters. If a test / tester
+is defined without an *ID*, it's *title* (Test) or *name* (Tester) attribute is used as an *ID*. However, in this case
+*tricot* does not check for duplicate *IDs* and you may end up with multiple tests / testers having the same *ID*.
+
+```yaml
+tester:
+  id: '001'
+  name: Basic Usage
+  description: |-
+    Demonstrate the basic usage of tricot
+
+tests:
+
+  - id: '001-1'
+    title: Test passwd File
+    description: |-
+      ...
+```
+
+To launch tests based on an *ID* you can use the command line switches ``--ids`` and ``--exclude-ids``. When using
+``--ids``, *tricot* only runs the tests / testers that match the specified *IDs*. If the *ID* belongs to a tester,
+all nested testers and tests are run, independent of their *ID*. The ``--exclude-ids`` can be used to exclude certain
+test / tester *IDs* from a test. Notice that ``--exclude-ids`` triggers before ``--ids``, so if you specify the same
+*ID* for both command line options, it is not run. On the other hand, this allows you to exclude nested test / tester
+*IDs* that are contained within a tester specified with the ``--ids`` option.
+
+*Test groups* can be used to group tests / testers together. Each test / tester definition can contain a ``groups`` key,
+which is a list within the *YAML* configuration. The contained items are the groups for the corresponding test / tester.
+*Test groups* are inherited from parent testers, but stacked instead of being merged together. E.g. when a parent tester
+is in the group `io` and the child tester in the group `logging`, the resulting group for the child tester is `io,logging`.
+
+```yaml
+# [io.yml]
+tester:
+  group:
+    - io
+  name: Test IO Modules
+  description: |-
+    Test IO Modules for the Software
+
+testers:
+  - ./nested.yml
+
+
+# [nested.yml]
+tester:
+  group:
+    - logging
+  name: Test IO Modules - logging
+  description: |-
+    Test IO Modules for the Software - logging
+
+tests:
+
+  title: Test Error Log
+  description: |-
+    ...
+```
+
+As for *IDs*, you can use the ``--groups`` and ``--exclude-groups`` command line options to run selective tests on *test
+groups*. However, group specifications on the command line support some special syntax. The easiest case is that you just
+want to run a single test group. E.g. taking the example above, to run the `io,logging` test you could use:
+
+```console
+tricot -v example.yml --groups io,logging
+```
+
+This is straight forwards, but it can get annoying if you defined `logging` groups in other parent testers than `io`.
+To make runs of a single test group, that is contained within different parent test groups easier, it is possible
+to specify wildcards.
+
+* `*` can be used to match an arbitrary group
+* `**` can be used to match an arbitrary number of arbitrary groups
+
+Running all tests from the `logging` group, independent of the parent test groups can be done like this:
+
+```console
+tricot -v example.yml --groups **,logging
+```
+
+In addition to wildcards, *tricot* also support *brace expressions*. These can be used to constructed *or-like* test
+groups. E.g. to run the `logging` module from the `io` and `networking` parent test groups, you could use:
+
+```console
+tricot -v example.yml --groups {io,networking},logging
+```
+
+Wildcards and *brace expressions* can also be used together within a group specifications. Whereas *brace expressions* can
+be placed at any location of a group specification, wildcards are not allowed within the last comma separated value. Also
+for group matching, the `--exclude-groups` option triggers before the `--groups` option.
+
+Both, *IDs* and *test groups* are case sensitive.
+
+
 ### Environment Variables
 
 ----

resources/bash_completion.d/tricot

@@ -12,7 +12,7 @@ function _tricot() {
     COMPREPLY=()
 
     # No completion
-	if _comp_contains "--exclude --numbers --testers --positionals --variables" $prev; then
+	if _comp_contains "--exclude-ids --exclude-groups --groups --ids --positionals --variables" $prev; then
 		return 0
 
     # Complete template modes
@@ -28,14 +28,15 @@ function _tricot() {
     elif [[ "$cur" == -* ]]; then
 		opts="--help"
 		opts="$opts --debug"
-		opts="$opts --exclude"
+		opts="$opts --exclude-ids"
+		opts="$opts --exclude-groups"
+		opts="$opts --groups"
+		opts="$opts --ids"
 		opts="$opts --logfile"
 		opts="$opts --load"
-		opts="$opts --numbers"
 		opts="$opts --positionals"
 		opts="$opts --quite"
 		opts="$opts --template"
-		opts="$opts --testers"
 		opts="$opts --variables"
 		opts="$opts --verbose"
 

setup.py

@@ -11,7 +11,7 @@
     url='https://github.com/qtc-de/tricot',
     name='tricot',
     author='Tobias Neitzel (@qtc_de)',
-    version='1.5.0',
+    version='1.6.0',
     author_email='',
 
     description='Trivial Command Testser',

tests/pytest/Misc/group_matching_test.py

@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+
+import tricot
+import pytest
+
+
+config_list = [['this,is,a,simple,group,def']]
+config_list.append(['now,lets,try,{conditional,orlike},group,def'])
+config_list.append(['{this,it},works,also,{with,using},more,than,one'])
+config_list.append(['multiple,lists', 'can,also,be,defined'])
+config_list.append(['{this,it},works', 'also,{with,using},orlike'])
+config_list.append(['lets,add,*,some,wildcards'])
+config_list.append(['lets,add,**,some,wildcards'])
+config_list.append(['lets,*,**,some,wildcards'])
+config_list.append(['**,some,wildcards'])
+
+match_list = [[([['this', 'is', 'a', 'simple', 'group', 'def']], True), ([['nope']], False)]]
+match_list.append([([['now', 'lets', 'try', 'conditional', 'group', 'def']], True),
+                   ([['now', 'lets', 'try', 'orlike', 'group', 'def']], True),
+                   ([['now', 'lets', 'tri', 'orlike', 'group', 'def']], False)])
+match_list.append([([['this', 'works', 'also', 'with', 'more', 'than', 'one']], True),
+                   ([['this', 'works', 'also', 'using', 'more', 'than', 'one']], True),
+                   ([['it', 'works', 'also', 'with', 'more', 'than', 'one']], True),
+                   ([['it', 'works', 'also', 'using', 'more', 'than', 'one']], True),
+                   ([['ot', 'works', 'also', 'using', 'more', 'than', 'one']], False)])
+match_list.append([([['aaaa', 'bbbb', 'cccc'], ['dddd', 'eeee']], False),
+                   ([['aaaa', 'bbbb', 'cccc'], ['multiple', 'lists']], True),
+                   ([['can', 'also', 'be', 'defined'], ['aaaa', 'bbbb']], True)])
+match_list.append([([['aaaa', 'bbbb', 'cccc'], ['dddd', 'eeee']], False),
+                   ([['this', 'works'], ['multiple', 'lists']], True),
+                   ([['also', 'using', 'orlike'], ['aaaa', 'bbbb']], True)])
+match_list.append([([['lets', 'add', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+                   ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], False)])
+match_list.append([([['lets', 'add', 'hi :)', 'hi :D', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+                   ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], False)])
+match_list.append([([['lets', 'add', 'hi :)', 'hi :D', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+                   ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True)])
+match_list.append([([['lets', 'add', 'hi :)', 'hi :D', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+                   ([['lets', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True),
+                   ([['aaaaa', 'hi :)', 'some', 'wildcards'], ['dddd', 'eeee']], True)])
+
+
+@pytest.mark.parametrize('config, match', zip(config_list, match_list))
+def test_group_matching(config, match):
+    '''
+    Check whether group matches are matching as expected
+    '''
+    for tupl in match:
+
+        groups = tricot.utils.parse_groups(config)
+        assert tricot.utils.groups_contain(groups, tupl[0]) == tupl[1]

tests/pytest/Misc/group_parsing_test.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python3
+
+import tricot
+import pytest
+
+
+config_list = [['this,is,a,simple,group,def']]
+config_list.append(['now,lets,try,{conditional,orlike},group,def'])
+config_list.append(['{this,it},works,also,{with,using},more,than,one'])
+config_list.append(['multiple,lists', 'can,also,be,defined'])
+config_list.append(['{this,it},works', 'also,{with,using},orlike'])
+
+result_list = [[['this', 'is', 'a', 'simple', 'group', 'def']]]
+result_list.append([['now', 'lets', 'try', 'conditional', 'group', 'def'],
+                   ['now', 'lets', 'try', 'orlike', 'group', 'def']])
+result_list.append([['this', 'works', 'also', 'with', 'more', 'than', 'one'],
+                   ['this', 'works', 'also', 'using', 'more', 'than', 'one'],
+                   ['it', 'works', 'also', 'with', 'more', 'than', 'one'],
+                   ['it', 'works', 'also', 'using', 'more', 'than', 'one']])
+result_list.append([['multiple', 'lists'], ['can', 'also', 'be', 'defined']])
+result_list.append([['this', 'works'], ['also', 'with', 'orlike'],
+                   ['this', 'works'], ['also', 'using', 'orlike'],
+                   ['it', 'works'], ['also', 'with', 'orlike'],
+                   ['it', 'works'], ['also', 'using', 'orlike']])
+
+
+@pytest.mark.parametrize('config, results', zip(config_list, result_list))
+def test_group_parsing(config, results):
+    '''
+    Check whether group specifications are parsed correctly.
+    '''
+    groups = tricot.utils.parse_groups(config)
+
+    for group in groups:
+        assert group in results

tests/tricot/test-cases/extra/Groups.yml

@@ -0,0 +1,9 @@
+tester:
+  name: groups
+  title: Group Tests
+  description: |-
+    "Performs different checks on tricot's test group feature"
+
+testers:
+  - Groups/nested_one.yml
+  - Groups/nested_two.yml