From feda72d884895de8e3bfee577c4b3573000955b5 Mon Sep 17 00:00:00 2001 From: Florian Weikert Date: Mon, 9 Dec 2019 14:51:35 -0800 Subject: [PATCH] Modify aggregate_... to file GitHub issues for flags bazelci.py now stores the used Bazel version as metadata. aggregate_incompatible_flags_test_result.py reads this value and files a GitHub issue for every project and every flag that needs migration. Fixes #869 --- ...ggregate_incompatible_flags_test_result.py | 167 +++++++++++++++++- buildkite/bazelci.py | 27 ++- 2 files changed, 184 insertions(+), 10 deletions(-) diff --git a/buildkite/aggregate_incompatible_flags_test_result.py b/buildkite/aggregate_incompatible_flags_test_result.py index 58cb817a1a..001af7e19d 100755 --- a/buildkite/aggregate_incompatible_flags_test_result.py +++ b/buildkite/aggregate_incompatible_flags_test_result.py @@ -17,6 +17,8 @@ import argparse import collections import os +import re +import requests import sys import threading @@ -30,6 +32,71 @@ FAIL_IF_MIGRATION_REQUIRED = os.environ.get("USE_BAZELISK_MIGRATE", "").upper() == "FAIL" +REPO_PATTERN = re.compile(r"https?://github.com/(?P[^/]+)/(?P[^/]+).git") + +ISSUE_TEMPLATE = """Incompatible flag {flag} will break {project} once Bazel {version} is released. + +Please see the following CI builds for more information: + +{links} + +Questions? Please file an issue in https://github.com/bazelbuild/continuous-integration +""" + +GITHUB_ISSUE_REPORTER = "bazel-flag-bot" + +GITHUB_TOKEN_KMS_KEY = "github-api-token" + +ENCRYPTED_GITHUB_API_TOKEN = """ +CiQA6OLsm2YFaO2fOFkdj3TCxCihvMNmf6HYKWXVSKnfDQtuYEsSUQBsAAJAI9UgPCsJZCQMC+QB/g4eFd +02IGzaOhSuCYyllc9Lr332wYAt7P52vXgmAU1zLfzGsm0iJ1KzjFW82BsYA6rgeSq4dCPTa8csRqND9Q== +""".strip() + + +class GitHubError(Exception): + def __init__(self, code, message): + super(GitHubError, self).__init__("{}: {}".format(code, message)) + self.code = code + self.message = message + + +class GitHubIssueClient(object): + def __init__(self, reporter, oauth_token): + self._reporter = reporter + self._session = requests.Session() + self._session.headers.update( + { + "Accept": "application/vnd.github.v3+json", + "Authorization": "token {}".format(oauth_token), + "Content-Type": "application/json", + } + ) + + def get_issue(self, repo_owner, repo_name, title): + # Returns an arbitrary matching issue if multiple matching issues exist. + json_data = self._send_request(repo_owner, repo_name, params={"creator": self._reporter}) + for i in json_data: + if i["title"] == title: + return i["number"] + + def create_issue(self, repo_owner, repo_name, title, body): + json_data = self._send_request( + repo_owner, + repo_name, + post=True, + json={"title": title, "body": body, "assignee": None, "labels": [], "milestone": None}, + ) + return json_data.get("number", "") + + def _send_request(self, repo_owner, repo_name, post=False, **kwargs): + url = "https://api.github.com/repos/{}/{}/issues".format(repo_owner, repo_name) + method = self._session.post if post else self._session.get + response = method(url, **kwargs) + if response.status_code // 100 != 2: + raise GitHubError(response.status_code, response.content) + + return response.json() + class LogFetcher(threading.Thread): def __init__(self, job, client): @@ -132,16 +199,17 @@ def print_projects_need_to_migrate(failed_jobs_per_flag): def print_flags_need_to_migrate(failed_jobs_per_flag): # The info box printed later is above info box printed before, # so reverse the flag list to maintain the same order. + printed_flag_boxes = False for flag in sorted(list(failed_jobs_per_flag.keys()), reverse=True): jobs = failed_jobs_per_flag[flag] if jobs: github_url = INCOMPATIBLE_FLAGS[flag] - info_text = [] - info_text.append(f"* **{flag}** " + get_html_link_text(":github:", github_url)) + info_text = [f"* **{flag}** " + get_html_link_text(":github:", github_url)] info_text += merge_and_format_jobs(jobs.values(), " - **{}**: {}") # Use flag as the context so that each flag gets a different info box. print_info(flag, "error", info_text) - if len(info_text) == 1: + printed_flag_boxes = True + if printed_flag_boxes: return info_text = ["#### Downstream projects need to migrate for the following flags:"] print_info("flags_need_to_migrate", "error", info_text) @@ -199,7 +267,7 @@ def print_info(context, style, info): ) -def print_result_info(build_number, client): +def analyze_logs(build_number, client): build_info = client.get_build_info(build_number) already_failing_jobs = [] @@ -219,6 +287,10 @@ def print_result_info(build_number, client): thread.join() process_build_log(failed_jobs_per_flag, already_failing_jobs, thread.log, thread.job) + return already_failing_jobs, failed_jobs_per_flag + + +def print_result_info(already_failing_jobs, failed_jobs_per_flag): print_flags_need_to_migrate(failed_jobs_per_flag) print_projects_need_to_migrate(failed_jobs_per_flag) @@ -226,10 +298,87 @@ def print_result_info(build_number, client): print_already_fail_jobs(already_failing_jobs) print_flags_ready_to_flip(failed_jobs_per_flag) - + return bool(failed_jobs_per_flag) +def notify_projects(failed_jobs_per_flag): + bazel_version = get_bazel_version() + if not bazel_version or bazel_version == "unreleased binary": + raise bazelci.BuildkiteException( + "Notification: Invalid Bazel version '{}'".format(bazel_version) + ) + + links_per_project_and_flag = collections.defaultdict(set) + for flag, job_data in failed_jobs_per_flag.items(): + for job in job_data.values(): + project_label, platform = get_pipeline_and_platform(job) + link = get_html_link_text(platform, job["web_url"]) + links_per_project_and_flag[(project_label, flag)].add("[{}]({})".format(platform, link)) + + try: + github_token = bazelci.decrypt_token( + encrypted_token=ENCRYPTED_GITHUB_API_TOKEN, kms_key=GITHUB_TOKEN_KMS_KEY + ) + except Exception as ex: + raise bazelci.BuildkiteException("Failed to decrypt GitHub API token: {}".format(ex)) + + errors = [] + issue_client = GitHubIssueClient(reporter=GITHUB_ISSUE_REPORTER, oauth_token=github_token) + for (project_label, flag), links in links_per_project_and_flag.items(): + try: + repo_owner, repo_name = get_project_owner_and_repo(project_label) + title = get_issue_title(project_label, bazel_version, flag) + if issue_client.get_issue(repo_owner, repo_name, title): + bazelci.eprint( + "There is already an issue in {}/{} for project {} and Bazel {}".format( + repo_owner, repo_name, project_label, bazel_version + ) + ) + else: + body = create_issue_body(project_label, bazel_version, flag, links) + issue_client.create_issue(repo_owner, repo_name, title, body) + except (bazelci.BuildkiteException, GitHubError) as ex: + errors.append("Could not notify project '{}': {}".format(project_label, ex)) + + if errors: + print_info("notify_errors", "error", errors) + + +def get_bazel_version(): + return bazelci.execute_command_and_get_output( + ["buildkite-agent", "meta-data", "get", bazelci.BAZEL_VERSION_METADATA_KEY, "--default", ""] + ) + + +def get_project_owner_and_repo(project_label): + full_repo = bazelci.DOWNSTREAM_PROJECTS.get(project_label, {}).get("git_repository", "") + if not full_repo: + raise bazelci.BuildkiteException( + "Could not retrieve Git repository for project '{}'".format(project_label) + ) + match = REPO_PATTERN.match(full_repo) + if not match: + raise bazelci.BuildkiteException( + "Hosts other than GitHub are currently not supported ({})".format(full_repo) + ) + + return match.group("owner"), match.group("repo") + + +def get_issue_title(project_label, bazel_version, flag): + return "Flag {} will break {} in Bazel {}".format(flag, project_label, bazel_version) + + +def create_issue_body(project_label, bazel_version, flag, links): + return ISSUE_TEMPLATE.format( + project=project_label, + version=bazel_version, + flag=flag, + links="\n".join("* {}".format(l) for l in links), + ) + + def main(argv=None): if argv is None: argv = sys.argv[1:] @@ -238,12 +387,18 @@ def main(argv=None): description="Script to aggregate `bazelisk --migrate` test result for incompatible flags and generate pretty Buildkite info messages." ) parser.add_argument("--build_number", type=str) + parser.add_argument("--notify", type=bool, nargs="?", const=True) args = parser.parse_args(argv) try: if args.build_number: client = bazelci.BuildkiteClient(org=BUILDKITE_ORG, pipeline=PIPELINE) - migration_required = print_result_info(args.build_number, client) + already_failing_jobs, failed_jobs_per_flag = analyze_logs(args.build_number, client) + migration_required = print_result_info(already_failing_jobs, failed_jobs_per_flag) + + if args.notify: + notify_projects(failed_jobs_per_flag) + if migration_required and FAIL_IF_MIGRATION_REQUIRED: bazelci.eprint("Exiting with code 3 since a migration is required.") return 3 diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index 2256435ebb..415d81e273 100755 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py @@ -511,6 +511,8 @@ CONFIG_FILE_EXTENSIONS = {".yml", ".yaml"} +BAZEL_VERSION_METADATA_KEY = "bazel_version" + class BuildkiteException(Exception): """ @@ -834,6 +836,7 @@ def execute_commands( execute_shell_commands(task_config.get("shell_commands", None)) bazel_version = print_bazel_version_info(bazel_binary, platform) + store_bazel_version(bazel_version) print_environment_variables_info() @@ -1036,6 +1039,15 @@ def print_bazel_version_info(bazel_binary, platform): return match.group(1) if match else "unreleased binary" +def store_bazel_version(bazel_version): + code = execute_command( + ["buildkite-agent", "meta-data", "set", BAZEL_VERSION_METADATA_KEY, bazel_version], + fail_if_nonzero=False, + ) + if code: + eprint("Unable to store Bazel version") + + def print_environment_variables_info(): print_collapsed_group(":information_source: Environment Variables") for key, value in os.environ.items(): @@ -1762,6 +1774,7 @@ def print_project_pipeline( monitor_flaky_tests, use_but, incompatible_flags, + notify, ): task_configs = configs.get("tasks", None) if not task_configs: @@ -1896,7 +1909,7 @@ def set_env_var(config_key, env_var_name): # the USE_BAZELISK_MIGRATE env var, but that are not being run as part of a # downstream pipeline. number = os.getenv("BUILDKITE_BUILD_NUMBER") - pipeline_steps += get_steps_for_aggregating_migration_results(number) + pipeline_steps += get_steps_for_aggregating_migration_results(number, notify) print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_project) @@ -2320,7 +2333,7 @@ def fetch_incompatible_flags(): def print_bazel_downstream_pipeline( - task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects + task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects, notify ): if not task_configs: raise BuildkiteException("Bazel downstream pipeline configuration is empty.") @@ -2380,7 +2393,7 @@ def print_bazel_downstream_pipeline( raise BuildkiteException("Not running inside Buildkite") if use_bazelisk_migrate(): pipeline_steps += get_steps_for_aggregating_migration_results( - current_build_number + current_build_number, notify ) else: pipeline_steps.append({"wait": "~", "continue_on_failure": "true"}) @@ -2420,12 +2433,14 @@ def print_bazel_downstream_pipeline( print_pipeline_steps(pipeline_steps) -def get_steps_for_aggregating_migration_results(current_build_number): +def get_steps_for_aggregating_migration_results(current_build_number, notify): parts = [ PLATFORMS[DEFAULT_PLATFORM]["python"], "aggregate_incompatible_flags_test_result.py", "--build_number=%s" % current_build_number, ] + if notify: + parts.append("--notify") return [ {"wait": "~", "continue_on_failure": "true"}, create_step( @@ -2748,6 +2763,7 @@ def main(argv=None): bazel_downstream_pipeline.add_argument( "--test_disabled_projects", type=bool, nargs="?", const=True ) + bazel_downstream_pipeline.add_argument("--notify", type=bool, nargs="?", const=True) project_pipeline = subparsers.add_parser("project_pipeline") project_pipeline.add_argument("--project_name", type=str) @@ -2757,6 +2773,7 @@ def main(argv=None): project_pipeline.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True) project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True) project_pipeline.add_argument("--incompatible_flag", type=str, action="append") + project_pipeline.add_argument("--notify", type=bool, nargs="?", const=True) runner = subparsers.add_parser("runner") runner.add_argument("--task", action="store", type=str, default="") @@ -2808,6 +2825,7 @@ def main(argv=None): file_config=args.file_config, test_incompatible_flags=args.test_incompatible_flags, test_disabled_projects=args.test_disabled_projects, + notify=args.notify, ) elif args.subparsers_name == "project_pipeline": configs = fetch_configs(args.http_config, args.file_config) @@ -2820,6 +2838,7 @@ def main(argv=None): monitor_flaky_tests=args.monitor_flaky_tests, use_but=args.use_but, incompatible_flags=args.incompatible_flag, + notify=args.notify, ) elif args.subparsers_name == "runner": configs = fetch_configs(args.http_config, args.file_config)