From e4407c4ed37b6c3d17cf7698f550cb39f627e4e3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 29 Oct 2021 11:48:21 -0400 Subject: [PATCH] Add a glob blacklist option for the build system - some variants are either not buildable (bugs) or want to restrict variants to build for build performance (#11200) --- integrations/cloudbuild/build-all.yaml | 4 +- integrations/cloudbuild/smoke-test.yaml | 4 +- scripts/build/build/targets.py | 35 +++++++++++++++-- scripts/build/build_examples.py | 13 +++++-- scripts/build/test.py | 7 ++++ .../testdata/all_targets_except_host.txt | 4 +- .../glob_star_targets_except_host.txt | 39 +++++++++++++++++++ 7 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 scripts/build/testdata/glob_star_targets_except_host.txt diff --git a/integrations/cloudbuild/build-all.yaml b/integrations/cloudbuild/build-all.yaml index 41d55cd5fe263c..4194d60d827268 100644 --- a/integrations/cloudbuild/build-all.yaml +++ b/integrations/cloudbuild/build-all.yaml @@ -17,8 +17,8 @@ steps: - PW_ENVIRONMENT_ROOT=/pwenv args: - >- - ./scripts/build/build_examples.py --enable-flashbundle build - --create-archives /workspace/artifacts/ + ./scripts/build/build_examples.py --enable-flashbundle + --target-glob '*' build --create-archives /workspace/artifacts/ id: CompileAll waitFor: - Bootstrap diff --git a/integrations/cloudbuild/smoke-test.yaml b/integrations/cloudbuild/smoke-test.yaml index 1518a963f93061..9205c9b08e51f2 100644 --- a/integrations/cloudbuild/smoke-test.yaml +++ b/integrations/cloudbuild/smoke-test.yaml @@ -86,8 +86,8 @@ steps: args: - >- ./scripts/build/build_examples.py --enable-flashbundle - --target-glob 'android-arm64-chip-tool' build - --create-archives /workspace/artifacts/ + --target-glob 'android-arm64-chip-tool' build --create-archives + /workspace/artifacts/ waitFor: - Bootstrap - Linux diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 40ec69f4d032cd..c0b5346be15169 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -37,8 +37,19 @@ class Target: def __init__(self, name, builder_class, **kwargs): self.name = name self.builder_class = builder_class + self.glob_blacklist_reason = None + self.create_kw_args = kwargs + def Clone(self): + """Creates a clone of self.""" + + clone = Target(self.name, self.builder_class, + **self.create_kw_args.copy()) + clone.glob_blacklist_reason = self.glob_blacklist_reason + + return clone + def Extend(self, suffix, **kargs): """Creates a clone of the current object extending its build parameters. @@ -46,8 +57,7 @@ def Extend(self, suffix, **kargs): suffix: appended with a "-" as separator to the clone name **kargs: arguments needed to produce the new build variant """ - clone = Target(self.name, self.builder_class, - **self.create_kw_args.copy()) + clone = self.Clone() clone.name += "-" + suffix clone.create_kw_args.update(kargs) return clone @@ -56,12 +66,31 @@ def Create(self, runner, repository_path: str, output_prefix: str, enable_flashb builder = self.builder_class( repository_path, runner=runner, **self.create_kw_args) + builder.target = self builder.identifier = self.name builder.output_dir = os.path.join(output_prefix, self.name) builder.enable_flashbundle(enable_flashbundle) return builder + def GlobBlacklist(self, reason): + clone = self.Clone() + if clone.glob_blacklist_reason: + clone.glob_blacklist_reason += ", " + clone.glob_blacklist_reason += reason + else: + clone.glob_blacklist_reason = reason + + return clone + + @property + def IsGlobBlacklisted(self): + return self.glob_blacklist_reason is not None + + @property + def GlobBlacklistReason(self): + return self.glob_blacklist_reason + def HostTargets(): target = Target(HostBoard.NATIVE.PlatformName(), HostBuilder) @@ -135,7 +164,7 @@ def NrfTargets(): for target in targets: yield target.Extend('lock', app=NrfApp.LOCK) yield target.Extend('light', app=NrfApp.LIGHT) - yield target.Extend('light-rpc', app=NrfApp.LIGHT, enable_rpcs=True) + yield target.Extend('light-rpc', app=NrfApp.LIGHT, enable_rpcs=True).GlobBlacklist('Compile failure due to pw_build args not forwarded to proto compiler. https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/66760') yield target.Extend('shell', app=NrfApp.SHELL) yield target.Extend('pump', app=NrfApp.PUMP) yield target.Extend('pump-controller', app=NrfApp.PUMP_CONTROLLER) diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py index 1d786cedf13c8c..c88194b11e8b9f 100755 --- a/scripts/build/build_examples.py +++ b/scripts/build/build_examples.py @@ -68,7 +68,7 @@ def ValidateRepoPath(context, parameter, value): type=click.Choice( ['all'] + [t.name for t in build.ALL_TARGETS], case_sensitive=False), multiple=True, - help='Build target(s)' + help='Build target(s). Note that "all" includes glob blacklisted targets' ) @click.option( '--target-glob', @@ -139,6 +139,8 @@ def main(context, log_level, target, target_glob, skip_target_glob, repo, out_pr runner = ShellRunner(root=repo) if 'all' in target: + # NOTE: ALL includes things that are glob blacklisted (so that 'targets' works and + # displays all) targets = build.ALL_TARGETS else: requested_targets = set([t.lower for t in target]) @@ -152,7 +154,8 @@ def main(context, log_level, target, target_glob, skip_target_glob, repo, out_pr if target_glob: matcher = GlobMatcher(target_glob) - targets = [t for t in targets if matcher.matches(t.name)] + targets = [t for t in targets if matcher.matches( + t.name) and not t.IsGlobBlacklisted] if skip_target_glob: matcher = GlobMatcher(skip_target_glob) @@ -184,7 +187,11 @@ def cmd_generate(context): @click.pass_context def cmd_generate(context): for builder in context.obj.builders: - print(builder.identifier) + if builder.target.IsGlobBlacklisted: + print("%s (NOGLOB: %s)" % + (builder.target.name, builder.target.GlobBlacklistReason)) + else: + print(builder.target.name) @main.command('build', help='generate and run ninja/make as needed to compile') diff --git a/scripts/build/test.py b/scripts/build/test.py index a179ec0a35af94..8128ad28525f63 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -106,6 +106,13 @@ def test_targets(self): '--skip-target-glob {linux,darwin}-* targets'.split(' ') ) + def test_glob_targets(self): + self.assertCommandOutput( + os.path.join('testdata', 'glob_star_targets_except_host.txt'), + '--target-glob * --skip-target-glob {linux,darwin}-* targets'.split( + ' ') + ) + @unittest.skipUnless(sys.platform == 'linux', 'Build on linux test') @unittest.skipUnless(os.uname().machine == 'x86_64', 'Validation x64 and crosscompile, requires linux x64') def test_linux_build(self): diff --git a/scripts/build/testdata/all_targets_except_host.txt b/scripts/build/testdata/all_targets_except_host.txt index 25200c8c0fda30..47b59abed64e97 100644 --- a/scripts/build/testdata/all_targets_except_host.txt +++ b/scripts/build/testdata/all_targets_except_host.txt @@ -25,13 +25,13 @@ esp32-m5stack-all-clusters-rpc esp32-m5stack-all-clusters-rpc-ipv6only infineon-p6-lock nrf-nrf52840-light -nrf-nrf52840-light-rpc +nrf-nrf52840-light-rpc (NOGLOB: Compile failure due to pw_build args not forwarded to proto compiler. https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/66760) nrf-nrf52840-lock nrf-nrf52840-pump nrf-nrf52840-pump-controller nrf-nrf52840-shell nrf-nrf5340-light -nrf-nrf5340-light-rpc +nrf-nrf5340-light-rpc (NOGLOB: Compile failure due to pw_build args not forwarded to proto compiler. https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/66760) nrf-nrf5340-lock nrf-nrf5340-pump nrf-nrf5340-pump-controller diff --git a/scripts/build/testdata/glob_star_targets_except_host.txt b/scripts/build/testdata/glob_star_targets_except_host.txt new file mode 100644 index 00000000000000..581495af81a2e3 --- /dev/null +++ b/scripts/build/testdata/glob_star_targets_except_host.txt @@ -0,0 +1,39 @@ +ameba-amebad-all-clusters +android-androidstudio-arm-chip-tool +android-androidstudio-arm64-chip-tool +android-androidstudio-x64-chip-tool +android-androidstudio-x86-chip-tool +android-arm-chip-tool +android-arm64-chip-test +android-arm64-chip-tool +android-x64-chip-tool +android-x86-chip-tool +efr32-brd4161a-light +efr32-brd4161a-light-rpc +efr32-brd4161a-lock +efr32-brd4161a-window-covering +esp32-c3devkit-all-clusters +esp32-devkitc-all-clusters +esp32-devkitc-all-clusters-ipv6only +esp32-devkitc-bridge +esp32-devkitc-lock +esp32-devkitc-shell +esp32-devkitc-temperature-measurement +esp32-m5stack-all-clusters +esp32-m5stack-all-clusters-ipv6only +esp32-m5stack-all-clusters-rpc +esp32-m5stack-all-clusters-rpc-ipv6only +infineon-p6-lock +nrf-nrf52840-light +nrf-nrf52840-lock +nrf-nrf52840-pump +nrf-nrf52840-pump-controller +nrf-nrf52840-shell +nrf-nrf5340-light +nrf-nrf5340-lock +nrf-nrf5340-pump +nrf-nrf5340-pump-controller +nrf-nrf5340-shell +qpg-qpg6100-lock +telink-tlsr9518adk80d-light +tizen-arm-light