diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index 2b3bcf3c4c..12e569f06b 100755 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py @@ -1170,6 +1170,14 @@ def use_bazelisk_migrate(): """ return is_trueish(os.environ.get("USE_BAZELISK_MIGRATE")) + +def is_downstream_pipeline(): + """ + Return true if BAZELCI_DOWNSTREAM_PIPELINE is set + """ + return is_trueish(os.environ.get("BAZELCI_DOWNSTREAM_PIPELINE")) + + def local_run_only(): """ If BAZELCI_LOCAL_RUN is set, run bazelci in local-only mode, with no attempt @@ -1991,6 +1999,12 @@ def common_build_flags(bep_file, platform): "--disk_cache=", ] + if is_downstream_pipeline(): + # If we are in a downstream pipeline, turn off the lockfile update since changing Bazel version could affect the lockfile. + flags.append("--lockfile_mode=off") + # Filter out targets that should not be built in downstream pipelines. + flags.append("--build_tag_filters=-no_bazel_downstream") + if is_windows(): pass elif is_mac(): @@ -2547,6 +2561,10 @@ def execute_bazel_test( "--build_tests_only", "--local_test_jobs=" + concurrent_test_jobs(platform), ] + if is_downstream_pipeline(): + # Filter out targets that should not be built in downstream pipelines. + aggregated_flags.append("--test_tag_filters=-no_bazel_downstream") + # Don't enable remote caching if the user enabled remote execution / caching themselves # or flaky test monitoring is enabled, as remote caching makes tests look less flaky than # they are. @@ -2789,12 +2807,9 @@ def print_project_pipeline( task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps) - # In Bazel Downstream Project pipelines, git_repository and project_name must be specified. - is_downstream_project = use_but and git_repository and project_name - buildifier_config = configs.get("buildifier") # Skip Buildifier when we test downstream projects. - if buildifier_config and not is_downstream_project: + if buildifier_config and not is_downstream_pipeline(): buildifier_env_vars = {} if isinstance(buildifier_config, str): # Simple format: @@ -2827,7 +2842,7 @@ def print_project_pipeline( # In Bazel Downstream Project pipelines, we should test the project at the last green commit. git_commit = None - if is_downstream_project: + if is_downstream_pipeline(): last_green_commit_url = bazelci_last_green_commit_url( git_repository, DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"] ) @@ -2844,7 +2859,7 @@ def print_project_pipeline( # only differ in the value of their explicit "bazel" field will be identical in the # downstream pipeline, thus leading to duplicate work. # Consequently, we filter those duplicate tasks here. - if is_downstream_project: + if is_downstream_pipeline(): h = hash_task_config(task, task_config) if h in config_hashes: skipped_downstream_tasks.append( @@ -2945,14 +2960,14 @@ def print_project_pipeline( if "validate_config" in configs: pipeline_steps += create_config_validation_steps() - if use_bazelisk_migrate() and not is_downstream_project: + if use_bazelisk_migrate() and not is_downstream_pipeline(): # Print results of bazelisk --migrate in project pipelines that explicitly set # the USE_BAZELISK_MIGRATE env var, but that are not being run as part of a # downstream pipeline. number = os.getenv("BUILDKITE_BUILD_NUMBER") pipeline_steps += get_steps_for_aggregating_migration_results(number, notify) - print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_project) + print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_pipeline()) def show_gerrit_review_link(git_repository, pipeline_steps): diff --git a/docs/downstream-testing.md b/docs/downstream-testing.md index 896909ad85..4666a070f5 100644 --- a/docs/downstream-testing.md +++ b/docs/downstream-testing.md @@ -30,6 +30,8 @@ The Bazel team monitors the downstream pipeline status and report issues for bre - The downstream project is expected to respond to the issue within 5 working days. Otherwise, the project is eligible to be temporarily disabled in the downstream pipeline. Note that, even if a pipeline is disabled from the [Bazel@HEAD + downstream](https://buildkite.com/bazel/bazel-at-head-plus-downstream) pipeline, the nightly result can still be checked from the [Bazel@HEAD+ Disabled](https://buildkite.com/bazel/bazel-at-head-plus-disabled) pipeline. - If a project remains disabled in the downstream pipeline for more than 6 months without any indication of a fix, we will remove the pipeline configuration from Bazel's downstream pipeline. +Note that: if you want to skip some builds in the downstream pipeline, you can specify `skip_in_bazel_downstream_pipeline: ` for a given job in your [Bazel CI configuration file](../buildkite/README.md#configuring-a-pipeline) or add `no_bazel_downstream` tag for certain build and test targets. + As of May 2023, some projects' pipeline config files live under [the "pipeline" directory](https://github.com/bazelbuild/continuous-integration/tree/master/pipelines) of this repository, which means the Bazel team is responsible for their setup for now, ideally they should be moved to their corresponding repository or the project should be removed. ## Testing Local Changes With All Downstream Projects