diff --git a/build/config/compiler/BUILD.gn b/build/config/compiler/BUILD.gn index 1796f02861c18e..909219fcffd24d 100644 --- a/build/config/compiler/BUILD.gn +++ b/build/config/compiler/BUILD.gn @@ -536,7 +536,18 @@ config("fuzzing_default") { } config("coverage") { - cflags = [ "--coverage" ] + if (is_clang) { + cflags = [ + # CLANG builds use the clang-specific coverage, so that versions of lcov/gcov + # do not have to match and `llvm-cov export -format=lcov` can be used + # + # Documentation: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html + "-fprofile-instr-generate", + "-fcoverage-mapping", + ] + } else { + cflags = [ "--coverage" ] + } ldflags = cflags } diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 3611e7372cb3c9..fd7a609cf70a97 100755 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -80,7 +80,7 @@ def BuildHostFakeTarget(): target.AppendModifier("pw-fuzztest", fuzzing_type=HostFuzzingType.PW_FUZZTEST).OnlyIfRe( "-clang").ExceptIfRe('-(libfuzzer|ossfuzz|asan)') target.AppendModifier('coverage', use_coverage=True).OnlyIfRe( - '-(chip-tool|all-clusters)') + '-(chip-tool|all-clusters)').ExceptIfRe('-clang') target.AppendModifier('dmalloc', use_dmalloc=True) target.AppendModifier('clang', use_clang=True) @@ -186,8 +186,7 @@ def BuildHostTarget(): "-clang").ExceptIfRe('-libfuzzer') target.AppendModifier("pw-fuzztest", fuzzing_type=HostFuzzingType.PW_FUZZTEST).OnlyIfRe( "-clang").ExceptIfRe('-(libfuzzer|ossfuzz|asan)') - target.AppendModifier('coverage', use_coverage=True).OnlyIfRe( - '-(chip-tool|all-clusters|tests)') + target.AppendModifier('coverage', use_coverage=True) target.AppendModifier('dmalloc', use_dmalloc=True) target.AppendModifier('clang', use_clang=True) target.AppendModifier('test', extra_tests=True) diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py index 0d6d99a5f00e71..8dd1cf3e93dd98 100644 --- a/scripts/build/builders/host.py +++ b/scripts/build/builders/host.py @@ -411,6 +411,7 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, if use_coverage: self.extra_gn_options.append('use_coverage=true') + self.use_clang = use_clang # for usage in other commands if use_clang: self.extra_gn_options.append('is_clang=true') @@ -586,7 +587,7 @@ def generate(self): self._Execute(['mkdir', '-p', self.coverage_dir], title="Create coverage output location") def PreBuildCommand(self): - if self.app == HostApp.TESTS and self.use_coverage: + if self.app == HostApp.TESTS and self.use_coverage and not self.use_clang: cmd = ['ninja', '-C', self.output_dir] if self.ninja_jobs is not None: @@ -604,7 +605,8 @@ def PreBuildCommand(self): '--output-file', os.path.join(self.coverage_dir, 'lcov_base.info')], title="Initial coverage baseline") def PostBuildCommand(self): - if self.app == HostApp.TESTS and self.use_coverage: + # TODO: CLANG coverage is not yet implemented, requires different tooling + if self.app == HostApp.TESTS and self.use_coverage and not self.use_clang: self._Execute(['lcov', '--capture', '--directory', os.path.join(self.output_dir, 'obj'), '--exclude', os.path.join(self.chip_dir, '**/tests/*'), '--exclude', os.path.join(self.chip_dir, 'zzz_generated/*'), diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py index bdb0680010c5b1..6531701a1b2f50 100644 --- a/scripts/tests/chiptest/test_definition.py +++ b/scripts/tests/chiptest/test_definition.py @@ -210,6 +210,36 @@ def items(self): self.microwave_oven_app, self.chip_repl_yaml_tester_cmd, self.chip_tool_with_python_cmd, self.rvc_app, self.network_manager_app] + def items_with_key(self): + """ + Returns all path items and also the corresponding "Application Key" which + is the typical application name. + + This is to provide scripts a consistent way to reference a path, even if + the paths used for individual appplications contain different names + (e.g. they could be wrapper scripts). + """ + return [ + (self.chip_tool, "chip-tool"), + (self.all_clusters_app, "chip-all-clusters-app"), + (self.lock_app, "chip-lock-app"), + (self.fabric_bridge_app, "fabric-bridge-app"), + (self.ota_provider_app, "chip-ota-provider-app"), + (self.ota_requestor_app, "chip-ota-requestor-app"), + (self.tv_app, "chip-tv-app"), + (self.bridge_app, "chip-bridge-app"), + (self.lit_icd_app, "lit-icd-app"), + (self.microwave_oven_app, "chip-microwave-oven-app"), + (self.chip_repl_yaml_tester_cmd, "yamltest_with_chip_repl_tester.py"), + ( + # This path varies, however it is a fixed python tool so it may be ok + self.chip_tool_with_python_cmd, + os.path.basename(self.chip_tool_with_python_cmd[-1]), + ), + (self.rvc_app, "chip-rvc-app"), + (self.network_manager_app, "matter-network-manager-app"), + ] + @dataclass class CaptureLine: @@ -330,7 +360,7 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, "don't know which application to run") if not dry_run: - for path in paths.items(): + for path, key in paths.items_with_key(): # Do not add chip-tool or chip-repl-yaml-tester-cmd to the register if path == paths.chip_tool or path == paths.chip_repl_yaml_tester_cmd or path == paths.chip_tool_with_python_cmd: continue @@ -345,8 +375,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, # For the app indicated by self.target, give it the 'default' key to add to the register if path == target_app: key = 'default' - else: - key = os.path.basename(path[-1]) app = App(runner, path) # Add the App to the register immediately, so if it fails during diff --git a/scripts/tests/local.py b/scripts/tests/local.py index eb9cbce8c0af68..df84777da3a344 100755 --- a/scripts/tests/local.py +++ b/scripts/tests/local.py @@ -14,10 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import enum import fnmatch import glob import logging +import multiprocessing import os import platform import shlex @@ -27,7 +29,7 @@ import textwrap import time from dataclasses import dataclass -from typing import List +from typing import List, Optional import alive_progress import click @@ -36,8 +38,6 @@ import tabulate import yaml -# We compile for the local architecture. Figure out what platform we need - def _get_native_machine_target(): """ @@ -55,6 +55,194 @@ def _get_native_machine_target(): return f"{current_system_info.system.lower()}-{arch}" +_CONFIG_PATH = "out/local_py.ini" + + +def get_coverage_default(coverage: Optional[bool]) -> bool: + if coverage is not None: + return coverage + config = configparser.ConfigParser() + try: + config.read(_CONFIG_PATH) + return config["OPTIONS"].getboolean("coverage") + except Exception: + return False + + +def _get_variants(coverage: Optional[bool]): + """ + compute the build variant suffixes for the given options + """ + variants = ["no-ble", "clang", "boringssl"] + + config = configparser.ConfigParser() + config["OPTIONS"] = {} + try: + config.read(_CONFIG_PATH) + logging.info("Defaults read from '%s'", _CONFIG_PATH) + except Exception: + config["OPTIONS"]["coverage"] = "true" + + if coverage is None: + # Coverage is NOT passed in as an explicit flag, so try to + # resume it from whatever last `build` flag was used + coverage = config["OPTIONS"].getboolean("coverage") + logging.info( + "Coverage setting not provided via command line. Will use: %s", coverage + ) + + if coverage: + variants.append("coverage") + config["OPTIONS"]["coverage"] = "true" + else: + config["OPTIONS"]["coverage"] = "false" + + if not os.path.exists("./out"): + os.mkdir("./out") + with open(_CONFIG_PATH, "w") as f: + config.write(f) + + return "-".join(variants) + + +@dataclass +class ApplicationTarget: + key: str # key for test_env running in python + target: str # target name for build_examples (and directory in out) + binary: str # elf binary to run after it is built + + +def _get_targets(coverage: Optional[bool]) -> list[ApplicationTarget]: + target_prefix = _get_native_machine_target() + suffix = _get_variants(coverage) + + targets = [] + + targets.append( + ApplicationTarget( + key="CHIP_TOOL", + target=f"{target_prefix}-chip-tool-{suffix}", + binary="chip-tool", + ) + ) + targets.append( + ApplicationTarget( + key="ALL_CLUSTERS_APP", + target=f"{target_prefix}-all-clusters-{suffix}", + binary="chip-all-clusters-app", + ) + ) + targets.append( + ApplicationTarget( + key="CHIP_LOCK_APP", + target=f"{target_prefix}-lock-{suffix}", + binary="chip-lock-app", + ) + ) + targets.append( + ApplicationTarget( + key="ENERGY_MANAGEMENT_APP", + target=f"{target_prefix}-energy-management-{suffix}", + binary="chip-energy-management-app", + ) + ) + targets.append( + ApplicationTarget( + key="LIT_ICD_APP", + target=f"{target_prefix}-lit-icd-{suffix}", + binary="lit-icd-app", + ) + ) + targets.append( + ApplicationTarget( + key="CHIP_MICROWAVE_OVEN_APP", + target=f"{target_prefix}-microwave-oven-{suffix}", + binary="chip-microwave-oven-app", + ) + ) + targets.append( + ApplicationTarget( + key="CHIP_RVC_APP", + target=f"{target_prefix}-rvc-{suffix}", + binary="chip-rvc-app", + ) + ) + targets.append( + ApplicationTarget( + key="NETWORK_MANAGEMENT_APP", + target=f"{target_prefix}-network-manager-ipv6only-{suffix}", + binary="matter-network-manager-app", + ) + ) + targets.append( + ApplicationTarget( + key="FABRIC_ADMIN_APP", + target=f"{target_prefix}-fabric-admin-no-wifi-rpc-ipv6only-{suffix}", + binary="fabric-admin", + ) + ) + targets.append( + ApplicationTarget( + key="FABRIC_BRIDGE_APP", + target=f"{target_prefix}-fabric-bridge-no-wifi-rpc-ipv6only-{suffix}", + binary="fabric-bridge-app", + ) + ) + targets.append( + ApplicationTarget( + key="FABRIC_SYNC_APP", + target=f"{target_prefix}-fabric-sync-no-wifi-ipv6only-{suffix}", + binary="fabric-sync", + ) + ) + targets.append( + ApplicationTarget( + key="LIGHTING_APP_NO_UNIQUE_ID", + target=f"{target_prefix}-light-data-model-no-unique-id-ipv6only-no-wifi-{suffix}", + binary="chip-lighting-app", + ) + ) + + # These are needed for chip tool tests + targets.append( + ApplicationTarget( + key="OTA_PROVIDER_APP", + target=f"{target_prefix}-ota-provider-{suffix}", + binary="chip-ota-provider-app", + ) + ) + targets.append( + ApplicationTarget( + key="OTA_REQUESTOR_APP", + target=f"{target_prefix}-ota-requestor-{suffix}", + binary="chip-ota-requestor-app", + ) + ) + targets.append( + ApplicationTarget( + key="TV_APP", + target=f"{target_prefix}-tv-app-{suffix}", + binary="chip-tv-app", + ) + ) + targets.append( + ApplicationTarget( + key="BRIDGE_APP", + target=f"{target_prefix}-bridge-{suffix}", + binary="chip-bridge-app", + ) + ) + targets.append( + ApplicationTarget( + key="TERMS_AND_CONDITIONS_APP", + target=f"{target_prefix}-terms-and-conditions-{suffix}", + binary="chip-terms-and-conditions-app", + ) + ) + + return targets + + class BinaryRunner(enum.Enum): """ Enumeration describing a wrapper runner for an application. Useful for debugging @@ -64,14 +252,22 @@ class BinaryRunner(enum.Enum): NONE = enum.auto() RR = enum.auto() VALGRIND = enum.auto() + COVERAGE = enum.auto() def execute_str(self, path: str): if self == BinaryRunner.NONE: return path elif self == BinaryRunner.RR: - return f"rr record {path}" + return f"exec rr record {path}" elif self == BinaryRunner.VALGRIND: - return f"valgrind {path}" + return f"exec valgrind {path}" + elif self == BinaryRunner.COVERAGE: + # Expected path is like "out//" + # + # We output up to 10K of separate profiles that will be merged as a + # final step. + rawname = os.path.basename(path[: path.rindex("/")] + "-run-%10000m.profraw") + return f'export LLVM_PROFILE_FILE="out/profiling/{rawname}"; exec {path}' __RUNNERS__ = { @@ -132,15 +328,19 @@ def cli(log_level): ) -def _with_activate(build_cmd: List[str]) -> List[str]: +def _with_activate(build_cmd: List[str], output_path=None) -> List[str]: """ Given a bash command list, will generate a new command suitable for subprocess with an execution of `scripts/activate.sh` prepended to it """ + cmd = shlex.join(build_cmd) + if output_path: + cmd = cmd + f" >{output_path}" + return [ "bash", "-c", - ";".join(["set -e", "source scripts/activate.sh", shlex.join(build_cmd)]), + ";".join(["set -e", "source scripts/activate.sh", cmd]) ] @@ -154,7 +354,7 @@ def _do_build_python(): ) -def _do_build_apps(): +def _do_build_apps(coverage: Optional[bool], ccache: bool): """ Builds example python apps suitable for running all python_tests. @@ -162,45 +362,32 @@ def _do_build_apps(): """ logging.info("Building example apps...") - target_prefix = _get_native_machine_target() - targets = [ - f"{target_prefix}-chip-tool-no-ble-clang-boringssl", - f"{target_prefix}-all-clusters-no-ble-clang-boringssl", - f"{target_prefix}-bridge-no-ble-clang-boringssl", - f"{target_prefix}-energy-management-no-ble-clang-boringssl", - f"{target_prefix}-fabric-admin-no-ble-no-wifi-rpc-ipv6only-clang-boringssl", - f"{target_prefix}-fabric-bridge-no-ble-no-wifi-rpc-ipv6only-clang-boringssl", - f"{target_prefix}-fabric-sync-no-ble-no-wifi-ipv6only-clang-boringssl", - f"{target_prefix}-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang", - f"{target_prefix}-lit-icd-no-ble-clang-boringssl", - f"{target_prefix}-lock-no-ble-clang-boringssl", - f"{target_prefix}-microwave-oven-no-ble-clang-boringssl", - f"{target_prefix}-network-manager-ipv6only-no-ble-clang-boringssl", - f"{target_prefix}-ota-provider-no-ble-clang-boringssl", - f"{target_prefix}-ota-requestor-no-ble-clang-boringssl", - f"{target_prefix}-rvc-no-ble-clang-boringssl", - f"{target_prefix}-tv-app-no-ble-clang-boringssl", - ] + targets = [t.target for t in _get_targets(coverage)] cmd = ["./scripts/build/build_examples.py"] for target in targets: cmd.append("--target") cmd.append(target) + + if ccache: + cmd.append("--pw-command-launcher=ccache") + cmd.append("build") subprocess.run(_with_activate(cmd), check=True) -def _do_build_basic_apps(): +def _do_build_basic_apps(coverage: Optional[bool]): """ Builds a minimal subset of test applications, specifically all-clusters and chip-tool only, for basic tests. """ logging.info("Building example apps...") - target_prefix = _get_native_machine_target() + + all_targets = dict([(t.key, t) for t in _get_targets(coverage)]) targets = [ - f"{target_prefix}-chip-tool-no-ble-clang-boringssl", - f"{target_prefix}-all-clusters-no-ble-clang-boringssl", + all_targets["CHIP_TOOL"].target, + all_targets["ALL_CLUSTERS_APP"].target, ] cmd = ["./scripts/build/build_examples.py"] @@ -213,9 +400,10 @@ def _do_build_basic_apps(): @cli.command() -def build_basic_apps(): +@click.option("--coverage/--no-coverage", default=None) +def build_basic_apps(coverage): """Builds chip-tool and all-clusters app.""" - _do_build_basic_apps() + _do_build_basic_apps(coverage) @cli.command() @@ -231,7 +419,9 @@ def build_python(): @cli.command() -def build_apps(): +@click.option("--coverage/--no-coverage", default=None) +@click.option("--ccache/--no-ccache", default=False) +def build_apps(coverage, ccache): """ Builds MANY apps used by python-tests. @@ -239,18 +429,20 @@ def build_apps(): To re-build the python environment use `build-python`. To re-build both python and apps, use `build` """ - _do_build_apps() + _do_build_apps(coverage, ccache) @cli.command() -def build(): +@click.option("--coverage/--no-coverage", default=None) +@click.option("--ccache/--no-ccache", default=False) +def build(coverage, ccache): """ Builds both python and apps (same as build-python + build-apps) Generally used together with `python-tests`. """ _do_build_python() - _do_build_apps() + _do_build_apps(coverage, ccache) def _maybe_with_runner(script_name: str, path: str, runner: BinaryRunner): @@ -262,14 +454,16 @@ def _maybe_with_runner(script_name: str, path: str, runner: BinaryRunner): return path # create a separate runner script based on the app - script_name = f"out/{script_name}.sh" + if not os.path.exists("out/runners"): + os.mkdir("out/runners") + + script_name = f"out/runners/{script_name}.sh" with open(script_name, "wt") as f: f.write( textwrap.dedent( f"""\ #!/usr/bin/env bash - - {runner.execute_str(path)} + {runner.execute_str(path)} "$@" """ ) ) @@ -294,12 +488,210 @@ def _add_target_to_cmd(cmd, flag, path, runner): cmd.append(_maybe_with_runner(flag[2:].replace("-", "_"), path, runner)) +@dataclass +class GlobFilter: + pattern: str + + def matches(self, txt: str) -> bool: + return fnmatch.fnmatch(txt, self.pattern) + + +@dataclass +class FilterList: + filters: list[GlobFilter] + + def any_matches(self, txt: str) -> bool: + return any([f.matches(txt) for f in self.filters]) + + +def _parse_filters(entry: str) -> FilterList: + if not entry: + entry = "*.*" + + if "," in entry: + entry_list = entry.split(",") + else: + entry_list = [entry] + + filters = [] + for f in entry_list: + if not f.startswith("*"): + f = "*" + f + if not f.endswith("*"): + f = f + "*" + filters.append(GlobFilter(pattern=f)) + + return FilterList(filters=filters) + + +@dataclass +class RawProfile: + raw_profile_paths: list[str] + binary_path: str + + +def _raw_profile_to_info(profile: RawProfile): + path = profile.raw_profile_paths[0] + + # Merge all per-app profiles into a single large profile + data_path = path[:path.find("-run-")] + ".profdata" + cmd = [ + "llvm-profdata", + "merge", + "-sparse", + ] + cmd.extend(["-o", data_path]) + cmd.extend(profile.raw_profile_paths) + + p = subprocess.run(_with_activate(cmd), check=True, capture_output=True) + + # Export to something lcov likes + cmd = [ + "llvm-cov", + "export", + "-format=lcov", + "--instr-profile", + data_path, + profile.binary_path + ] + + # for -ignore-filename-regex + ignore_paths = [ + "third_party/boringssl/.*", + "third_party/perfetto/.*", + "third_party/jsoncpp/.*", + "third_party/editline/.*", + "third_party/initpp/.*", + "third_party/libwebsockets/.*", + "third_party/pigweed/.*", + "third_party/nanopb/.*", + "third_party/nl.*", + "/usr/include/.*", + "/usr/lib/.*", + ] + + for p in ignore_paths: + cmd.append("-ignore-filename-regex") + cmd.append(p) + + info_path = path.replace(".profraw", ".info") + subprocess.run(_with_activate(cmd, output_path=info_path), check=True) + logging.info("Generated %s", info_path) + + # !!!!! HACK ALERT !!!!! + # + # The paths for our examples are generally including CHIP as + # examples//third_party/connectedhomeip/.... + # So we will replace every occurence of these to remove the extra indirection into third_party + # + # Generally we will replace every path (Shown as SF:...) with the corresponding item + # + # We assume that the info lines fit in RAM + lines = [] + with open(info_path, 'rt') as f: + for line in f.readlines(): + line = line.rstrip() + if line.startswith("SF:"): + # This is a source file line: "SF:..." + path = line[3:] + line = f"SF:{os.path.realpath(path)}\n" + lines.append(line) + + # re-write it. + with open(info_path, 'wt') as f: + f.write("\n".join(lines)) + + return info_path + + +@cli.command() +@click.option( + "--flat", + default=False, + is_flag=True, + show_default=True, + help="Use a flat (1-directory) layout rather than hierarchical.", +) +def gen_coverage(flat): + """ + Generate coverage from tests run with "--coverage" + """ + # This assumes default.profraw exists, so it tries to + # generate coverage out of it + # + # Each target gets its own profile + + raw_profiles = [] + for t in _get_targets(coverage=True): + glob_str = os.path.join("./out", "profiling", f"{t.target}*.profraw") + app_profiles = [] + for path in glob.glob(glob_str): + app_profiles.append(path) + + if app_profiles: + raw_profiles.append(RawProfile( + raw_profile_paths=app_profiles, + binary_path=os.path.join("./out", t.target, t.binary) + )) + else: + logging.warning("No profiles for %s", t.target) + + with multiprocessing.Pool() as p: + trace_files = p.map(_raw_profile_to_info, raw_profiles) + + if not trace_files: + logging.error( + "Could not find any trace files. Did you run tests with coverage enabled?" + ) + return + + cmd = ["lcov"] + for t in trace_files: + cmd.append("--add-tracefile") + cmd.append(t) + + errors_to_ignore = [ + "inconsistent", "range", "corrupt", "category" + ] + + cmd.append("--output-file") + cmd.append("out/profiling/merged.info") + for e in errors_to_ignore: + cmd.append("--ignore-errors") + cmd.append(e) + + if os.path.exists("out/profiling/merged.info"): + os.unlink("out/profiling/merged.info") + + subprocess.run(cmd, check=True) + + logging.info("Generating HTML...") + cmd = ["genhtml"] + for e in errors_to_ignore: + cmd.append("--ignore-errors") + cmd.append(e) + + cmd.append("--flat" if flat else "--hierarchical") + cmd.append("--output-directory") + cmd.append("out/coverage") + cmd.append("out/profiling/merged.info") + + subprocess.run(cmd, check=True) + logging.info("Coverage HTML should be available in out/coverage/index.html") + + @cli.command() @click.option( "--test-filter", default="*", show_default=True, - help="Run only tests that match the given glob filter.", + help="Run only tests that match the given glob filter(s). Comma separated list of filters", +) +@click.option( + "--skip", + default="", + show_default=True, + help="Skip the tests matching the given glob. Comma separated list of filters. Empty for no skipping.", ) @click.option( "--from-filter", @@ -335,8 +727,9 @@ def _add_target_to_cmd(cmd, flag, path, runner): "--fail-log-dir", default=None, help="Save failure logs into the specified directory instead of logging (as logging can be noisy/slow)", - type=click.Path(exists=True, file_okay=False, dir_okay=True), + type=click.Path(file_okay=False, dir_okay=True), ) +@click.option("--coverage/--no-coverage", default=None) @click.option( "--runner", default="none", @@ -345,12 +738,14 @@ def _add_target_to_cmd(cmd, flag, path, runner): ) def python_tests( test_filter, + skip, from_filter, from_skip_filter, dry_run, no_show_timings, runner, keep_going, + coverage, fail_log_dir, ): """ @@ -361,44 +756,35 @@ def python_tests( """ runner = __RUNNERS__[runner] + # make sure we are fully aware if running with or without coverage + coverage = get_coverage_default(coverage) + if coverage: + if runner != BinaryRunner.NONE: + logging.error("Runner for coverage is implict") + sys.exit(1) + + # wrap around so we get a good LLVM_PROFILE_FILE + runner = BinaryRunner.COVERAGE + def as_runner(path): return _maybe_with_runner(os.path.basename(path), path, runner) # create an env file - target_prefix = _get_native_machine_target() - with open("out/test_env.yaml", "wt") as f: - f.write( - textwrap.dedent( - f"""\ - ALL_CLUSTERS_APP: {as_runner(f'out/{target_prefix}-all-clusters-no-ble-clang-boringssl/chip-all-clusters-app')} - CHIP_LOCK_APP: {as_runner(f'out/{target_prefix}-lock-no-ble-clang-boringssl/chip-lock-app')} - ENERGY_MANAGEMENT_APP: { - as_runner(f'out/{target_prefix}-energy-management-no-ble-clang-boringssl/chip-energy-management-app')} - LIT_ICD_APP: {as_runner(f'out/{target_prefix}-lit-icd-no-ble-clang-boringssl/lit-icd-app')} - CHIP_MICROWAVE_OVEN_APP: { - as_runner(f'out/{target_prefix}-microwave-oven-no-ble-clang-boringssl/chip-microwave-oven-app')} - CHIP_RVC_APP: {as_runner(f'out/{target_prefix}-rvc-no-ble-clang-boringssl/chip-rvc-app')} - NETWORK_MANAGEMENT_APP: { - as_runner(f'out/{target_prefix}-network-manager-ipv6only-no-ble-clang-boringssl/matter-network-manager-app')} - FABRIC_ADMIN_APP: { - as_runner(f'out/{target_prefix}-fabric-admin-no-ble-no-wifi-rpc-ipv6only-clang-boringssl/fabric-admin')} - FABRIC_BRIDGE_APP: { - as_runner(f'out/{target_prefix}-fabric-bridge-no-ble-no-wifi-rpc-ipv6only-clang-boringssl/fabric-bridge-app')} - FABRIC_SYNC_APP: { - as_runner(f'out/{target_prefix}-fabric-sync-no-ble-no-wifi-ipv6only-clang-boringssl/fabric-sync')} - LIGHTING_APP_NO_UNIQUE_ID: {as_runner(f'out/{target_prefix}-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang/chip-lighting-app')} - TERMS_AND_CONDITIONS_APP: {as_runner(f'out/{target_prefix}-terms-and-conditions/chip-terms-and-conditions-app')} - TRACE_APP: out/trace_data/app-{{SCRIPT_BASE_NAME}} - TRACE_TEST_JSON: out/trace_data/test-{{SCRIPT_BASE_NAME}} - TRACE_TEST_PERFETTO: out/trace_data/test-{{SCRIPT_BASE_NAME}} - """ - ) - ) - - if not test_filter.startswith("*"): - test_filter = "*" + test_filter - if not test_filter.endswith("*"): - test_filter = test_filter + "*" + with open("./out/test_env.yaml", "wt") as f: + for target in _get_targets(coverage): + run_path = as_runner(f"out/{target.target}/{target.binary}") + f.write(f"{target.key}: {run_path}\n") + f.write("TRACE_APP: out/trace_data/app-{SCRIPT_BASE_NAME}\n") + f.write("TRACE_TEST_JSON: out/trace_data/test-{SCRIPT_BASE_NAME}\n") + f.write("TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME}\n") + + if not test_filter: + test_filter = "*" + test_filter = _parse_filters(test_filter) + + if skip: + print("SKIP IS %r" % skip) + skip = _parse_filters(skip) if from_filter: if not from_filter.startswith("*"): @@ -416,6 +802,9 @@ def as_runner(path): if not os.path.exists("out/trace_data"): os.mkdir("out/trace_data") + if fail_log_dir and not os.path.exists(fail_log_dir): + os.mkdir(fail_log_dir) + metadata = yaml.full_load(open("src/python_testing/test_metadata.yaml")) excluded_patterns = set([item["name"] for item in metadata["not_automated"]]) @@ -441,7 +830,7 @@ def as_runner(path): failed_tests = [] try: to_run = [] - for script in fnmatch.filter(test_scripts, test_filter or "*.*"): + for script in [t for t in test_scripts if test_filter.any_matches(t)]: if from_filter: if not fnmatch.fnmatch(script, from_filter): logging.info("From-filter SKIP %s", script) @@ -453,6 +842,11 @@ def as_runner(path): from_skip_filter = None logging.info("From-skip-filter SKIP %s", script) continue + if skip: + if skip.any_matches(script): + logging.info("EXPLICIT SKIP %s", script) + continue + to_run.append(script) with alive_progress.alive_bar(len(to_run), title="Running tests") as bar: @@ -475,13 +869,6 @@ def as_runner(path): base_name, slow_test_duration[base_name], ) - elif base_name == "TC_EEVSE_2_3.py": - # TODO: this should be fixed ... - # for now just note that a `TZ=UTC` makes this pass - logging.warning( - "Test %s is TIMEZONE dependent. Passes with UTC but fails on EST. If this fails set 'TZ=UTC' for running the test.", - base_name, - ) tstart = time.time() result = subprocess.run(cmd, capture_output=True) @@ -499,7 +886,7 @@ def as_runner(path): with open(out_name, "wb") as f: f.write(result.stdout) with open(err_name, "wb") as f: - f.write(result.stdout) + f.write(result.stderr) else: logging.info("STDOUT:\n%s", result.stdout.decode("utf8")) @@ -547,15 +934,16 @@ def as_runner(path): sys.exit(1) -def _do_build_fabric_sync_apps(): +def _do_build_fabric_sync_apps(coverage: Optional[bool]): """ Build applications used for fabric sync tests """ target_prefix = _get_native_machine_target() + suffix = _get_variants(coverage) targets = [ - f"{target_prefix}-fabric-admin-no-ble-no-wifi-rpc-ipv6only-clang-boringssl", - f"{target_prefix}-fabric-bridge-no-ble-no-wifi-rpc-ipv6only-clang-boringssl", - f"{target_prefix}-all-clusters-boringssl-no-ble", + f"{target_prefix}-fabric-admin-no-wifi-rpc-ipv6only-{suffix}", + f"{target_prefix}-fabric-bridge-no-wifi-rpc-ipv6only-{suffix}", + f"{target_prefix}-all-clusters-{suffix}", ] build_cmd = ["./scripts/build/build_examples.py"] @@ -568,21 +956,23 @@ def _do_build_fabric_sync_apps(): @cli.command() -def build_fabric_sync_apps(): +@click.option("--coverage/--no-coverage", default=None) +def build_fabric_sync_apps(coverage): """ Build fabric synchronizatio applications. """ - _do_build_fabric_sync_apps() + _do_build_fabric_sync_apps(coverage) @cli.command() -def build_fabric_sync(): +@click.option("--coverage/--no-coverage", default=None) +def build_fabric_sync(coverage): """ Builds both python environment and fabric sync applications """ # fabric sync interfaces with python for convenience, so do that _do_build_python() - _do_build_fabric_sync_apps() + _do_build_fabric_sync_apps(coverage) @cli.command() @@ -669,13 +1059,23 @@ def casting_test(test, log_directory, tv_app, tv_casting_app, runner): @click.option("--target-glob", default=None) @click.option("--include-tags", default=None) @click.option("--expected-failures", default=None) +@click.option("--coverage/--no-coverage", default=None) +@click.option( + "--keep-going", + default=False, + is_flag=True, + show_default=True, + help="Keep going on errors. Will report all failed tests at the end.", +) @click.option( "--runner", default="none", type=click.Choice(list(__RUNNERS__.keys()), case_sensitive=False), help="Determines the verbosity of script output", ) -def chip_tool_tests(target, target_glob, include_tags, expected_failures, runner): +def chip_tool_tests( + target, target_glob, include_tags, expected_failures, coverage, keep_going, runner +): """ Run integration tests using chip-tool. @@ -689,20 +1089,38 @@ def chip_tool_tests(target, target_glob, include_tags, expected_failures, runner # docker run --rm -it -v ~/devel/connectedhomeip:/workspace --privileged ghcr.io/project-chip/chip-build-vscode:64 runner = __RUNNERS__[runner] + # make sure we are fully aware if running with or without coverage + coverage = get_coverage_default(coverage) + if coverage: + if runner != BinaryRunner.NONE: + logging.error("Runner for coverage is implict") + sys.exit(1) + + # wrap around so we get a good LLVM_PROFILE_FILE + runner = BinaryRunner.COVERAGE + cmd = [ "./scripts/tests/run_test_suite.py", "--runner", "chip_tool_python", ] - target_prefix = _get_native_machine_target() - cmd.extend( - [ - "--chip-tool", - f"./out/{target_prefix}-chip-tool-no-ble-clang-boringssl/chip-tool", - ] + cmd.extend(["--exclude-tags", "MANUAL"]) + cmd.extend(["--exclude-tags", "IN_DEVELOPMENT"]) + cmd.extend(["--exclude-tags", "FLAKY"]) + cmd.extend(["--exclude-tags", "EXTRA_SLOW"]) + cmd.extend(["--exclude-tags", "PURPOSEFUL_FAILURE"]) + + paths = dict( + [(t.key, f"./out/{t.target}/{t.binary}") for t in _get_targets(coverage)] ) + if runner == BinaryRunner.COVERAGE: + # when running with coveage, chip-tool also is covered + cmd.extend(["--chip-tool", _maybe_with_runner("chip-tool", paths["CHIP_TOOL"], runner)]) + else: + cmd.extend(["--chip-tool", paths["CHIP_TOOL"]]) + if target is not None: cmd.extend(["--target", target]) @@ -714,65 +1132,32 @@ def chip_tool_tests(target, target_glob, include_tags, expected_failures, runner cmd.append("run") cmd.extend(["--iterations", "1"]) - cmd.extend(["--test-timeout-seconds", "60"]) + + # NOTE: we allow all runs here except extra slow + # This means our timeout is quite large + cmd.extend(["--test-timeout-seconds", "300"]) if expected_failures is not None: - cmd.extend(["--expected-failures", expected_failures, "--keep-going"]) + cmd.extend(["--expected-failures", expected_failures]) + keep_going = True # if we expect failures, we have to keep going + + if keep_going: + cmd.append("--keep-going") + + target_flags = [ + ("--all-clusters-app", "ALL_CLUSTERS_APP"), + ("--lock-app", "CHIP_LOCK_APP"), + ("--ota-provider-app", "OTA_PROVIDER_APP"), + ("--ota-requestor-app", "OTA_REQUESTOR_APP"), + ("--tv-app", "TV_APP"), + ("--bridge-app", "BRIDGE_APP"), + ("--lit-icd-app", "LIT_ICD_APP"), + ("--microwave-oven-app", "CHIP_MICROWAVE_OVEN_APP"), + ("--rvc-app", "CHIP_RVC_APP"), + ] - _add_target_to_cmd( - cmd, - "--all-clusters-app", - f"./out/{target_prefix}-all-clusters-no-ble-clang-boringssl/chip-all-clusters-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--lock-app", - f"./out/{target_prefix}-lock-no-ble-clang-boringssl/chip-lock-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--ota-provider-app", - f"./out/{target_prefix}-ota-provider-no-ble-clang-boringssl/chip-ota-provider-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--ota-requestor-app", - f"./out/{target_prefix}-ota-requestor-no-ble-clang-boringssl/chip-ota-requestor-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--tv-app", - f"./out/{target_prefix}-tv-app-no-ble-clang-boringssl/chip-tv-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--bridge-app", - f"./out/{target_prefix}-bridge-no-ble-clang-boringssl/chip-bridge-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--lit-icd-app", - f"./out/{target_prefix}-lit-icd-no-ble-clang-boringssl/lit-icd-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--microwave-oven-app", - f"./out/{target_prefix}-microwave-oven-no-ble-clang-boringssl/chip-microwave-oven-app", - runner, - ) - _add_target_to_cmd( - cmd, - "--rvc-app", - f"./out/{target_prefix}-rvc-no-ble-clang-boringssl/chip-rvc-app", - runner, - ) + for flag, path_key in target_flags: + _add_target_to_cmd(cmd, flag, paths[path_key], runner) subprocess.run(_with_activate(cmd), check=True) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py index 5a6481d3d315a1..d999a0e73122d7 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py @@ -1166,6 +1166,11 @@ def write_to_app_pipe(self, command_dict: dict, app_pipe_name: Optional[str] = N dut_ip = os.getenv('LINUX_DUT_IP') if dut_ip is None: + if not os.path.exists(app_pipe_name): + # Named pipes are unique, so we MUST have consistent PID/paths + # set up for them to work. + logging.error("Named pipe %r does NOT exist" % app_pipe_name) + raise FileNotFoundError("CANNOT FIND %r" % app_pipe_name) with open(app_pipe_name, "w") as app_pipe: app_pipe.write(command + "\n") # TODO(#31239): remove the need for sleep