[runtime tests] add support for multiple expects

This extends the runtime test functionality to allow
for multiple expects (EXPECT, EXPECT_NOT, EXPECT_JSON, etc...)
and requires that they ALL pass for the test to pass.

Example single expect failure:
```
[  FAILED  ] basic.it only lists probes in the program
	Command: /home/parallels/Desktop/shared/repos/bpftrace/build/src/bpftrace -l -e 'kfunc:vmlinux:vfs_read { exit(); }'
	Expected no REGEX: kfunc:vmlinux:vfs_read
	Found:
kfunc:vmlinux:vfs_read\n
```

Example multiple expect failures:
```
[  FAILED  ] basic.it only lists probes in the program
	Command: /home/parallels/Desktop/shared/repos/bpftrace/build/src/bpftrace -l -e 'kfunc:vmlinux:vfs_read { exit(); }'
	Expected no REGEX: kfunc:vmlinux:vfs_read
	Found:
kfunc:vmlinux:vfs_read\n
	Expected REGEX: kfunc:vmlinux:vfs_write
	Found:
kfunc:vmlinux:vfs_read\n
```

tests/README.md

@@ -65,7 +65,8 @@ Each runtime testcase consists of multiple directives. In no particular order:
   `RUN` unless you must pass flags or create a shell pipeline.  This XOR the
   `RUN` field is required
 * `EXPECT`: The expected output. Python regular expressions are supported.
-   One of EXPECT, EXPECT_NONE, EXPECT_FILE or EXPECT_JSON is required.
+   One or more `EXPECT` and `EXPECT_NONE` or a single `EXPECT_FILE` or 
+   `EXPECT_JSON` is required.
 * `EXPECT_NONE`: The negation of `EXPECT`, which also supports Python regular 
    expressions.
 * `EXPECT_FILE`: A file containing the expected output, matched as plain

tests/runtime/basic

@@ -135,8 +135,9 @@ EXPECT rawtracepoint:
 TIMEOUT 1
 
 NAME it only lists probes in the program
-RUN {{BPFTRACE}} -l -e 'kfunc:vmlinux:vfs_read { exit(); }' | grep kfunc | wc -l
-EXPECT 1
+RUN {{BPFTRACE}} -l -e 'kfunc:vmlinux:vfs_read { exit(); }'
+EXPECT kfunc:vmlinux:vfs_read
+EXPECT_NONE kfunc:vmlinux:vfs_write
 TIMEOUT 5
 
 NAME it lists uprobes in the program

tests/runtime/engine/parser.py

@@ -16,15 +16,19 @@ class UnknownFieldError(Exception):
 class InvalidFieldError(Exception):
     pass
 
+class Expect:
+  def __init__(self, expect, mode):
+    self.expect = expect
+    self.mode = mode
 
 TestStruct = namedtuple(
     'TestStruct',
     [
         'name',
         'run',
         'prog',
-        'expect',
-        'expect_mode', # regex, text, json ...
+        'expects',
+        'has_exact_expect',
         'timeout',
         'befores',
         'after',
@@ -102,8 +106,8 @@ def __read_test_struct(test, test_suite):
         name = ''
         run = ''
         prog = ''
-        expect = ''
-        expect_mode = ''
+        expects = []
+        has_exact_expect = False
         timeout = ''
         befores = []
         after = ''
@@ -130,17 +134,15 @@ def __read_test_struct(test, test_suite):
             elif item_name == "PROG":
                 prog = line
             elif item_name == 'EXPECT':
-                expect = line
-                expect_mode = 'regex'
+                expects.append(Expect(line, 'regex'))
             elif item_name == 'EXPECT_NONE':
-                expect = line
-                expect_mode = 'regex_none'
+                expects.append(Expect(line, 'regex_none'))
             elif item_name == 'EXPECT_FILE':
-                expect = line
-                expect_mode = 'file'
+                has_exact_expect = True
+                expects.append(Expect(line, 'file'))
             elif item_name == 'EXPECT_JSON':
-                expect = line
-                expect_mode = 'json'
+                has_exact_expect = True
+                expects.append(Expect(line, 'json'))
             elif item_name == 'TIMEOUT':
                 timeout = int(line.strip(' '))
             elif item_name == 'BEFORE':
@@ -207,17 +209,19 @@ def __read_test_struct(test, test_suite):
             raise RequiredFieldError('Test RUN or PROG is required. Suite: ' + test_suite)
         elif run != '' and prog != '':
             raise InvalidFieldError('Test RUN and PROG both specified. Suit: ' + test_suite)
-        elif expect == '':
-            raise RequiredFieldError('Test EXPECT is required. Suite: ' + test_suite)
+        elif len(expects) == 0:
+            raise RequiredFieldError('At leat one test EXPECT (or variation) is required. Suite: ' + test_suite)
+        elif len(expects) > 1 and has_exact_expect:
+            raise InvalidFieldError('EXPECT_JSON or EXPECT_FILE can not be used with other EXPECTs. Suite: ' + test_suite)
         elif timeout == '':
             raise RequiredFieldError('Test TIMEOUT is required. Suite: ' + test_suite)
 
         return TestStruct(
             name,
             run,
             prog,
-            expect,
-            expect_mode,
+            expects,
+            has_exact_expect,
             timeout,
             befores,
             after,

tests/runtime/engine/runner.py

@@ -98,7 +98,7 @@ def prepare_bpf_call(test, nsenter=[]):
 
             return nsenter_prefix + ret
         else:  # PROG
-            use_json = "-q -f json" if test.expect_mode == "json" else ""
+            use_json = "-q -f json" if test.expects[0].mode == "json" else ""
             cmd = nsenter_prefix + "{} {} -e '{}'".format(BPFTRACE_BIN, use_json, test.prog)
             # We're only reusing PROG-directive tests for AOT tests
             if test.suite == 'aot':
@@ -197,25 +197,36 @@ def run_test(test):
         # the primary RUN or PROG command, and the AFTER.
         nsenter = []
         bpf_call = "[unknown]"
+        failed_expects = []
 
         def get_pid_ns_cmd(cmd):
             return nsenter + [os.path.abspath(x) for x in cmd.split()]
-
-        def check_result(output):
+            
+        def check_expect(expect, output):
             try:
-                if test.expect_mode == "regex":
-                    return re.search(test.expect, output, re.M)
-                elif test.expect_mode == "regex_none":
-                    return not re.search(test.expect, output, re.M)
-                elif test.expect_mode == "file":
-                    # remove leading and trailing empty lines
-                    return output.strip() == open(test.expect).read().strip()
+                if expect.mode == "regex":
+                    return re.search(expect.expect, output, re.M)
+                elif expect.mode == "regex_none":
+                    return not re.search(expect.expect, output, re.M)
+                elif expect.mode == "file":
+                    with open(expect.expect) as expect_file:
+                        # remove leading and trailing empty lines
+                        return output.strip() == expect_file.read().strip()
                 else:
-                    return json.loads(output) == json.load(open(test.expect))
+                    with open(expect.expect) as expect_file:
+                        return json.loads(output) == json.load(expect_file)
             except Exception as err:
                 print("ERROR in check_result: ", err)
                 return False
 
+        def check_result(output):
+            all_passed = True
+            for expect in test.expects:
+                if not check_expect(expect, output):
+                    all_passed = False
+                    failed_expects.append(expect)
+            return all_passed
+
         try:
             result = None
             timeout = False
@@ -340,7 +351,7 @@ def found_all_children(lines):
                 output += nextline
                 if not attached and nextline == "__BPFTRACE_NOTIFY_PROBES_ATTACHED\n":
                     attached = True
-                    if test.expect_mode != "regex" and test.expect_mode != "regex_none":
+                    if test.has_exact_expect:
                         output = ""  # ignore earlier ouput
                     signal.alarm(test.timeout or DEFAULT_TIMEOUT)
                     if test.after:
@@ -440,25 +451,26 @@ def print_befores_and_after_output():
         else:
             print(fail("[  FAILED  ] ") + "%s.%s" % (test.suite, test.name))
             print('\tCommand: ' + bpf_call)
-            if test.expect_mode == "regex":
-                print('\tExpected REGEX: ' + test.expect)
-                print('\tFound:\n' + to_utf8(output))
-            elif test.expect_mode == "regex_none":
-                print('\tExpected no REGEX: ' + test.expect)
-                print('\tFound:\n' + to_utf8(output))
-            elif test.expect_mode == "json":
-                try:
-                    expected = json.dumps(json.loads(open(test.expect).read()), indent=2)
-                except json.decoder.JSONDecodeError as err:
-                    expected = "Could not parse JSON: " + str(err)
-                try:
-                    found = json.dumps(json.loads(output), indent=2)
-                except json.decoder.JSONDecodeError as err:
-                    found = "Could not parse JSON: " + str(err)
-                print('\tExpected JSON:\n' + expected)
-                print('\tFound:\n' + found)
-            else:
-                print('\tExpected FILE:\n\t\t' + to_utf8(open(test.expect).read()))
-                print('\tFound:\n\t\t' + to_utf8(output))
+            for failed_expect in failed_expects:
+                if failed_expect.mode == "regex":
+                    print('\tExpected REGEX: ' + failed_expect.expect)
+                    print('\tFound:\n' + to_utf8(output))
+                elif failed_expect.mode == "regex_none":
+                    print('\tExpected no REGEX: ' + failed_expect.expect)
+                    print('\tFound:\n' + to_utf8(output))
+                elif failed_expect.mode == "json":
+                    try:
+                        expected = json.dumps(json.loads(open(failed_expect.expect).read()), indent=2)
+                    except json.decoder.JSONDecodeError as err:
+                        expected = "Could not parse JSON: " + str(err)
+                    try:
+                        found = json.dumps(json.loads(output), indent=2)
+                    except json.decoder.JSONDecodeError as err:
+                        found = "Could not parse JSON: " + str(err)
+                    print('\tExpected JSON:\n' + expected)
+                    print('\tFound:\n' + found)
+                else:
+                    print('\tExpected FILE:\n\t\t' + to_utf8(open(failed_expect.expect).read()))
+                    print('\tFound:\n\t\t' + to_utf8(output))
             print_befores_and_after_output()
             return Runner.FAIL