Merge pull request #17 from qtc-de/develop

Prepare v1.9.0 Release
qtc-de · Dec 26, 2021 · b0e7113 · b0e7113
2 parents 907a593 + 2cf2cd2
commit b0e7113
Show file tree

Hide file tree

Showing 11 changed files with 453 additions and 129 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [1.9.0] - Dec 26, 2021
+
+### Added
+
+* Testers can now specify [external requirements](https://github.com/qtc-de/tricot/tree/develop/docs#external-requirements)
+* Requirements are checked before the tester is run and can contain:
+  * Required files (including checksums verification)
+  * Required operating system commands
+  * Required tricot version
+
+### Changed
+
+* Improved the selective test functionalities (`--skip-until`, `--continue-from`, `--ids`, `--groups`, `--exclude`, `--exclude-groups`).
+  Skipped tests are now skipped completely and they neither appear in the output nor start their associated containers.
+* Fix bug related to KeyboardInterrupts 
+* Update test cases to make them more compatible among different distributions
+
+
 ## [1.8.0] - Dec 04, 2021
 
 ### Changed

diff --git a/bin/tricot b/bin/tricot
@@ -200,7 +200,16 @@ def main():
 
         try:
             tester = tricot.Tester.from_file(yml_file, runtime_vars=variables)
-            tester.run(set(args.ids), groups, set(args.eids), egroups)
+            tester.filter(set(args.ids), groups, set(args.eids), egroups)
+            tester.check_requirements()
+            tester.run()
+
+        except KeyboardInterrupt:
+            tricot.Logger.reset_indent()
+            tricot.Logger.print('')
+            tricot.Logger.print_mixed_yellow('Caught', 'KeyboardInterrupt', 'from user.')
+            tricot.Logger.print('Stopping current test.')
+            sys.exit(tricot.constants.KEYBOARD_INTERRUPT)
 
         except Exception as excpt:
 
@@ -319,6 +328,21 @@ def main():
                     tricot.Logger.print_with_indent_blue(str(e), e=True)
                     sys.exit(tricot.constants.PARSER_ERROR)
 
+                except tricot.TricotRequiredFile as e:
+                    tricot.Logger.print_mixed_yellow('Error: Test configuration requires missing file:', str(e), e=True)
+                    tricot.Logger.print_mixed_blue('Affected configuration:', wrapper.path, e=True)
+                    sys.exit(tricot.constants.MISSING_RESOURCE)
+
+                except tricot.TricotRequiredCommand as e:
+                    tricot.Logger.print_mixed_yellow('Error: Test configuration requires missing command:', str(e), e=True)
+                    tricot.Logger.print_mixed_blue('Affected configuration:', wrapper.path, e=True)
+                    sys.exit(tricot.constants.MISSING_RESOURCE)
+
+                except tricot.TricotVersionMismatch as e:
+                    tricot.Logger.print_mixed_yellow('Error: Test configuration requires different tricot version:', str(e), e=True)
+                    tricot.Logger.print_mixed_blue('Affected configuration:', wrapper.path, e=True)
+                    sys.exit(tricot.constants.VERSION_MISMATCH)
+
                 except Exception as e:
                     tricot.Logger.print_mixed_yellow('Caught', 'unexpected Exception', 'while running tricot.', e=True)
                     tricot.Logger.increase_indent()
@@ -332,13 +356,6 @@ def main():
                 tricot.Logger.print_with_indent_blue(str(e), e=True)
                 sys.exit(tricot.constants.YAML_SYNTAX_ERROR)
 
-            except KeyboardInterrupt:
-                tricot.Logger.reset_indent()
-                tricot.Logger.print('')
-                tricot.Logger.print_mixed_yellow('Caught', 'KeyboardInterrupt', 'from user.')
-                tricot.Logger.print('Stopping current test.')
-                sys.exit(tricot.constants.KEYBOARD_INTERRUPT)
-
             except Exception as e:
                 tricot.Logger.print_mixed_yellow('Caught', 'unexpected Exception', 'while running tricot.', e=True)
                 tricot.Logger.increase_indent()

diff --git a/docs/README.md b/docs/README.md
@@ -20,6 +20,7 @@ on it's own.
 - [Conditionals](#conditionals)
 - [Reusing Output](#reusing-output)
 - [Logging](#logging)
+- [External Requirements](#external-requirements)
 - [Custom Strings](#custom-strings)
 - [Worth Knowing](#worth-knowing)
 
@@ -801,6 +802,53 @@ Log files are always written in verbose mode and contain the full details for ea
 This is also true, even if the corresponding *test* or *tester* run was successful.
 
 
+### External Requirements
+
+----
+
+*tricot* allows you to use arbitrary resources on your system for testing. This can make tests incompatible
+across different platforms. To prevent errors at runtime, you can specify some of the external requirements
+within your test configuration. *tricot* checks these requirements first before running the tests. Currently,
+you can require certain files to exist, certain commands to exist and a specific version of tricot to run the
+test. All this needs to be configured within the tester configuration:
+
+```yaml
+tester:
+  title: Basic Usage
+  description: |-
+    Demonstrate the basic usage of tricot
+
+  requires:
+    files:
+      - /etc/passwd
+    commands:
+      - cat
+    tricot:
+      eq: 1.9.0
+      le: 1.9.0
+      lt: 1.9.0
+      ge: 1.9.0
+      gt: 1.9.0
+```
+
+File based requirements can also include a checksum for the specified file:
+
+```yaml
+tester:
+  title: Basic Usage
+  description: |-
+    Demonstrate the basic usage of tricot
+
+  requires:
+    files:
+      - filename: /etc/passwd
+        md5: c6beb132462d61bdd851de604acec9c7
+        sha1: 6de989b32cb10f2361ddaa46ea917a674429b4c6
+        sha256: f5aa7815387c6f8bad54554b5632a775f9c95cedcf4400b3f78395d4e2f59c0f
+        sha512: c26f20ee2d251198d189b53d4f3437769b4381dcf8d53c7e445740de333b2e671a2133932fb2089e2d90ec7eef78af3fefbe28d0d6b6d5dbdaf5a121705ed347
+```
+
+
 ### Custom Strings
 
 ----

diff --git a/setup.py b/setup.py
@@ -11,7 +11,7 @@
     url='https://github.com/qtc-de/tricot',
     name='tricot',
     author='Tobias Neitzel (@qtc_de)',
-    version='1.8.0',
+    version='1.9.0',
     author_email='',
 
     description='Trivial Command Testser',

diff --git a/tests/tricot/test-cases/validator-tests/CountValidator.yml b/tests/tricot/test-cases/validator-tests/CountValidator.yml
@@ -14,11 +14,11 @@ tests:
     validators:
       - count:
           values:
-            - backup
-            - nobody
+            - :0:0
+            - systemd-network
           counts:
-            - 3
-            - 2
+            - 1
+            - 1
 
   - title: Case Insensitive - Success
     description: >
@@ -30,11 +30,11 @@ tests:
       - count:
           ignore_case: True
           values:
-            - BackUP
-            - nObOdy
+            - :0:0
+            - SystemD-NetworK
           counts:
-            - 3
-            - 2
+            - 1
+            - 1
 
   - title: Check Failure - Fail
     description: >
@@ -46,8 +46,8 @@ tests:
       - count:
           ignore_case: False
           values:
-            - BackUP
-            - nObOdy
+            - :0:0
+            - SystemD-NetworK
           counts:
-            - 3
-            - 2
+            - 1
+            - 1
diff --git a/tests/tricot/test-cases/validator-tests/RegexValidator.yml b/tests/tricot/test-cases/validator-tests/RegexValidator.yml
@@ -66,7 +66,7 @@ tests:
           ignore_case: True
           dotall: True
           match:
-            - ^rOOt.+sh.daemon
+            - ^rOOt.+sh.+daemon
 
   - title: Invert - Fail
     description: >

diff --git a/tests/tricot/tricot-tests/extra-tests.yml b/tests/tricot/tricot-tests/extra-tests.yml
@@ -1016,10 +1016,11 @@ tests:
             - '[003-001] Dummy Test 2... success'
             - '[003-002] Dummy Test 3... success'
             - '[003-003] Dummy Test 4... success'
-            - '[001-000] Dummy Test 1... skipped'
-            - '[001-001] Dummy Test 2... skipped'
-            - '[001-002] Dummy Test 3... skipped'
-            - '[001-003] Dummy Test 4... skipped'
+          invert:
+            - '[001-000] Dummy Test 1... success'
+            - '[001-001] Dummy Test 2... success'
+            - '[001-002] Dummy Test 3... success'
+            - '[001-003] Dummy Test 4... success'
 
 
   - title: Skip Until 3
@@ -1045,13 +1046,14 @@ tests:
             - '[003-001] Dummy Test 2... success'
             - '[003-002] Dummy Test 3... success'
             - '[003-003] Dummy Test 4... success'
-            - '[001-000] Dummy Test 1... skipped'
-            - '[001-001] Dummy Test 2... skipped'
-            - '[001-002] Dummy Test 3... skipped'
-            - '[001-003] Dummy Test 4... skipped'
-            - '[002-000] Dummy Test 1... skipped'
-            - '[002-001] Dummy Test 2... skipped'
-            - '[002-002] Dummy Test 3... skipped'
+          invert:
+            - '[001-000] Dummy Test 1... success'
+            - '[001-001] Dummy Test 2... success'
+            - '[001-002] Dummy Test 3... success'
+            - '[001-003] Dummy Test 4... success'
+            - '[002-000] Dummy Test 1... success'
+            - '[002-001] Dummy Test 2... success'
+            - '[002-002] Dummy Test 3... success'
 
 
   - title: Continue From 1
@@ -1077,13 +1079,14 @@ tests:
             - '[003-001] Dummy Test 2... success'
             - '[003-002] Dummy Test 3... success'
             - '[003-003] Dummy Test 4... success'
-            - '[001-000] Dummy Test 1... skipped'
-            - '[001-001] Dummy Test 2... skipped'
-            - '[001-002] Dummy Test 3... skipped'
-            - '[001-003] Dummy Test 4... skipped'
-            - '[002-000] Dummy Test 1... skipped'
-            - '[002-001] Dummy Test 2... skipped'
-            - '[002-002] Dummy Test 3... skipped'
+          invert:
+            - '[001-000] Dummy Test 1... success'
+            - '[001-001] Dummy Test 2... success'
+            - '[001-002] Dummy Test 3... success'
+            - '[001-003] Dummy Test 4... success'
+            - '[002-000] Dummy Test 1... success'
+            - '[002-001] Dummy Test 2... success'
+            - '[002-002] Dummy Test 3... success'
 
 
   - title: Continue From 2
@@ -1112,10 +1115,11 @@ tests:
             - '[003-001] Dummy Test 2... success'
             - '[003-002] Dummy Test 3... success'
             - '[003-003] Dummy Test 4... success'
-            - '[001-000] Dummy Test 1... skipped'
-            - '[001-001] Dummy Test 2... skipped'
-            - '[001-002] Dummy Test 3... skipped'
-            - '[001-003] Dummy Test 4... skipped'
+          invert:
+            - '[001-000] Dummy Test 1... success'
+            - '[001-001] Dummy Test 2... success'
+            - '[001-002] Dummy Test 3... success'
+            - '[001-003] Dummy Test 4... success'
 
 
   - title: ID Pattern

diff --git a/tests/tricot/tricot.yml b/tests/tricot/tricot.yml
@@ -14,7 +14,4 @@ variables:
   EXTRA: ../test-cases/extra/
 
 testers:
-  - ./tricot-tests/plugin-tests.yml
-  - ./tricot-tests/validator-tests.yml
-  - ./tricot-tests/extractor-tests.yml
-  - ./tricot-tests/extra-tests.yml
+  - ./tricot-tests/*
diff --git a/tricot/constants.py b/tricot/constants.py
@@ -19,6 +19,8 @@
 EXTRACTOR_ERROR = 18
 DUPLICATE_ID_ERROR = 19
 YAML_SYNTAX_ERROR = 20
+MISSING_RESOURCE = 21
+VERSION_MISMATCH = 22
 
 LAST_ERROR = 0
-VERSION = '1.8.0'
+VERSION = '1.9.0'