Modify aggregate_... to file GitHub issues for flags

bazelci.py now stores the used Bazel version as metadata.
aggregate_incompatible_flags_test_result.py reads this value and files a GitHub issue for every project and every flag that needs migration.

Fixes #869

buildkite/aggregate_incompatible_flags_test_result.py

@@ -17,6 +17,8 @@
 import argparse
 import collections
 import os
+import re
+import requests
 import sys
 import threading
 
@@ -30,6 +32,71 @@
 
 FAIL_IF_MIGRATION_REQUIRED = os.environ.get("USE_BAZELISK_MIGRATE", "").upper() == "FAIL"
 
+REPO_PATTERN = re.compile(r"https?://github.com/(?P<owner>[^/]+)/(?P<repo>[^/]+).git")
+
+ISSUE_TEMPLATE = """Incompatible flag {flag} will break {project} once Bazel {version} is released.
+
+Please see the following CI builds for more information:
+
+{links}
+
+Questions? Please file an issue in https://github.com/bazelbuild/continuous-integration
+"""
+
+GITHUB_ISSUE_REPORTER = "bazel-flag-bot"
+
+GITHUB_TOKEN_KMS_KEY = "github-api-token"
+
+ENCRYPTED_GITHUB_API_TOKEN = """
+CiQA6OLsm2YFaO2fOFkdj3TCxCihvMNmf6HYKWXVSKnfDQtuYEsSUQBsAAJAI9UgPCsJZCQMC+QB/g4eFd
+02IGzaOhSuCYyllc9Lr332wYAt7P52vXgmAU1zLfzGsm0iJ1KzjFW82BsYA6rgeSq4dCPTa8csRqND9Q==
+""".strip()
+
+
+class GitHubError(Exception):
+    def __init__(self, code, message):
+        super(GitHubError, self).__init__("{}: {}".format(code, message))
+        self.code = code
+        self.message = message
+
+
+class GitHubIssueClient(object):
+    def __init__(self, reporter, oauth_token):
+        self._reporter = reporter
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                "Accept": "application/vnd.github.v3+json",
+                "Authorization": "token {}".format(oauth_token),
+                "Content-Type": "application/json",
+            }
+        )
+
+    def get_issue(self, repo_owner, repo_name, title):
+        # Returns an arbitrary matching issue if multiple matching issues exist.
+        json_data = self._send_request(repo_owner, repo_name, params={"creator": self._reporter})
+        for i in json_data:
+            if i["title"] == title:
+                return i["number"]
+
+    def create_issue(self, repo_owner, repo_name, title, body):
+        json_data = self._send_request(
+            repo_owner,
+            repo_name,
+            post=True,
+            json={"title": title, "body": body, "assignee": None, "labels": [], "milestone": None},
+        )
+        return json_data.get("number", "")
+
+    def _send_request(self, repo_owner, repo_name, post=False, **kwargs):
+        url = "https://api.github.com/repos/{}/{}/issues".format(repo_owner, repo_name)
+        method = self._session.post if post else self._session.get
+        response = method(url, **kwargs)
+        if response.status_code // 100 != 2:
+            raise GitHubError(response.status_code, response.content)
+
+        return response.json()
+
 
 class LogFetcher(threading.Thread):
     def __init__(self, job, client):
@@ -132,16 +199,17 @@ def print_projects_need_to_migrate(failed_jobs_per_flag):
 def print_flags_need_to_migrate(failed_jobs_per_flag):
     # The info box printed later is above info box printed before,
     # so reverse the flag list to maintain the same order.
+    printed_flag_boxes = False
     for flag in sorted(list(failed_jobs_per_flag.keys()), reverse=True):
         jobs = failed_jobs_per_flag[flag]
         if jobs:
             github_url = INCOMPATIBLE_FLAGS[flag]
-            info_text = []
-            info_text.append(f"* **{flag}** " + get_html_link_text(":github:", github_url))
+            info_text = [f"* **{flag}** " + get_html_link_text(":github:", github_url)]
             info_text += merge_and_format_jobs(jobs.values(), "  - **{}**: {}")
             # Use flag as the context so that each flag gets a different info box.
             print_info(flag, "error", info_text)
-    if len(info_text) == 1:
+            printed_flag_boxes = True
+    if printed_flag_boxes:
         return
     info_text = ["#### Downstream projects need to migrate for the following flags:"]
     print_info("flags_need_to_migrate", "error", info_text)
@@ -199,7 +267,7 @@ def print_info(context, style, info):
         )
 
 
-def print_result_info(build_number, client):
+def analyze_logs(build_number, client):
     build_info = client.get_build_info(build_number)
 
     already_failing_jobs = []
@@ -219,17 +287,98 @@ def print_result_info(build_number, client):
         thread.join()
         process_build_log(failed_jobs_per_flag, already_failing_jobs, thread.log, thread.job)
 
+    return already_failing_jobs, failed_jobs_per_flag
+
+
+def print_result_info(already_failing_jobs, failed_jobs_per_flag):
     print_flags_need_to_migrate(failed_jobs_per_flag)
 
     print_projects_need_to_migrate(failed_jobs_per_flag)
 
     print_already_fail_jobs(already_failing_jobs)
 
     print_flags_ready_to_flip(failed_jobs_per_flag)
-    
+
     return bool(failed_jobs_per_flag)
 
 
+def notify_projects(failed_jobs_per_flag):
+    bazel_version = get_bazel_version()
+    if not bazel_version or bazel_version == "unreleased binary":
+        raise bazelci.BuildkiteException(
+            "Notification: Invalid Bazel version '{}'".format(bazel_version)
+        )
+
+    links_per_project_and_flag = collections.defaultdict(set)
+    for flag, job_data in failed_jobs_per_flag.items():
+        for job in job_data.values():
+            project_label, platform = get_pipeline_and_platform(job)
+            link = get_html_link_text(platform, job["web_url"])
+            links_per_project_and_flag[(project_label, flag)].add("[{}]({})".format(platform, link))
+
+    try:
+        github_token = bazelci.decrypt_token(
+            encrypted_token=ENCRYPTED_GITHUB_API_TOKEN, kms_key=GITHUB_TOKEN_KMS_KEY
+        )
+    except Exception as ex:
+        raise bazelci.BuildkiteException("Failed to decrypt GitHub API token: {}".format(ex))
+
+    errors = []
+    issue_client = GitHubIssueClient(reporter=GITHUB_ISSUE_REPORTER, oauth_token=github_token)
+    for (project_label, flag), links in links_per_project_and_flag.items():
+        try:
+            repo_owner, repo_name = get_project_owner_and_repo(project_label)
+            title = get_issue_title(project_label, bazel_version, flag)
+            if issue_client.get_issue(repo_owner, repo_name, title):
+                bazelci.eprint(
+                    "There is already an issue in {}/{} for project {} and Bazel {}".format(
+                        repo_owner, repo_name, project_label, bazel_version
+                    )
+                )
+            else:
+                body = create_issue_body(project_label, bazel_version, flag, links)
+                issue_client.create_issue(repo_owner, repo_name, title, body)
+        except (bazelci.BuildkiteException, GitHubError) as ex:
+            errors.append("Could not notify project '{}': {}".format(project_label, ex))
+
+    if errors:
+        print_info("notify_errors", "error", errors)
+
+
+def get_bazel_version():
+    return bazelci.execute_command_and_get_output(
+        ["buildkite-agent", "meta-data", "get", bazelci.BAZEL_VERSION_METADATA_KEY, "--default", ""]
+    )
+
+
+def get_project_owner_and_repo(project_label):
+    full_repo = bazelci.DOWNSTREAM_PROJECTS.get(project_label, {}).get("git_repository", "")
+    if not full_repo:
+        raise bazelci.BuildkiteException(
+            "Could not retrieve Git repository for project '{}'".format(project_label)
+        )
+    match = REPO_PATTERN.match(full_repo)
+    if not match:
+        raise bazelci.BuildkiteException(
+            "Hosts other than GitHub are currently not supported ({})".format(full_repo)
+        )
+
+    return match.group("owner"), match.group("repo")
+
+
+def get_issue_title(project_label, bazel_version, flag):
+    return "Flag {} will break {} in Bazel {}".format(flag, project_label, bazel_version)
+
+
+def create_issue_body(project_label, bazel_version, flag, links):
+    return ISSUE_TEMPLATE.format(
+        project=project_label,
+        version=bazel_version,
+        flag=flag,
+        links="\n".join("* {}".format(l) for l in links),
+    )
+
+
 def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]
@@ -238,12 +387,18 @@ def main(argv=None):
         description="Script to aggregate `bazelisk --migrate` test result for incompatible flags and generate pretty Buildkite info messages."
     )
     parser.add_argument("--build_number", type=str)
+    parser.add_argument("--notify", type=bool, nargs="?", const=True)
 
     args = parser.parse_args(argv)
     try:
         if args.build_number:
             client = bazelci.BuildkiteClient(org=BUILDKITE_ORG, pipeline=PIPELINE)
-            migration_required = print_result_info(args.build_number, client)
+            already_failing_jobs, failed_jobs_per_flag = analyze_logs(args.build_number, client)
+            migration_required = print_result_info(already_failing_jobs, failed_jobs_per_flag)
+
+            if args.notify:
+                notify_projects(failed_jobs_per_flag)
+
             if migration_required and FAIL_IF_MIGRATION_REQUIRED:
                 bazelci.eprint("Exiting with code 3 since a migration is required.")
                 return 3

buildkite/bazelci.py

@@ -511,6 +511,8 @@
 
 CONFIG_FILE_EXTENSIONS = {".yml", ".yaml"}
 
+BAZEL_VERSION_METADATA_KEY = "bazel_version"
+
 
 class BuildkiteException(Exception):
     """
@@ -834,6 +836,7 @@ def execute_commands(
             execute_shell_commands(task_config.get("shell_commands", None))
 
         bazel_version = print_bazel_version_info(bazel_binary, platform)
+        store_bazel_version(bazel_version)
 
         print_environment_variables_info()
 
@@ -1036,6 +1039,15 @@ def print_bazel_version_info(bazel_binary, platform):
     return match.group(1) if match else "unreleased binary"
 
 
+def store_bazel_version(bazel_version):
+    code = execute_command(
+        ["buildkite-agent", "meta-data", "set", BAZEL_VERSION_METADATA_KEY, bazel_version],
+        fail_if_nonzero=False,
+    )
+    if code:
+        eprint("Unable to store Bazel version")
+
+
 def print_environment_variables_info():
     print_collapsed_group(":information_source: Environment Variables")
     for key, value in os.environ.items():
@@ -1762,6 +1774,7 @@ def print_project_pipeline(
     monitor_flaky_tests,
     use_but,
     incompatible_flags,
+    notify,
 ):
     task_configs = configs.get("tasks", None)
     if not task_configs:
@@ -1896,7 +1909,7 @@ def set_env_var(config_key, env_var_name):
         # the USE_BAZELISK_MIGRATE env var, but that are not being run as part of a
         # downstream pipeline.
         number = os.getenv("BUILDKITE_BUILD_NUMBER")
-        pipeline_steps += get_steps_for_aggregating_migration_results(number)
+        pipeline_steps += get_steps_for_aggregating_migration_results(number, notify)
 
     print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_project)
 
@@ -2320,7 +2333,7 @@ def fetch_incompatible_flags():
 
 
 def print_bazel_downstream_pipeline(
-    task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects
+    task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects, notify
 ):
     if not task_configs:
         raise BuildkiteException("Bazel downstream pipeline configuration is empty.")
@@ -2380,7 +2393,7 @@ def print_bazel_downstream_pipeline(
             raise BuildkiteException("Not running inside Buildkite")
         if use_bazelisk_migrate():
             pipeline_steps += get_steps_for_aggregating_migration_results(
-                current_build_number
+                current_build_number, notify
             )
         else:
             pipeline_steps.append({"wait": "~", "continue_on_failure": "true"})
@@ -2420,12 +2433,14 @@ def print_bazel_downstream_pipeline(
     print_pipeline_steps(pipeline_steps)
 
 
-def get_steps_for_aggregating_migration_results(current_build_number):
+def get_steps_for_aggregating_migration_results(current_build_number, notify):
     parts = [
         PLATFORMS[DEFAULT_PLATFORM]["python"],
         "aggregate_incompatible_flags_test_result.py",
         "--build_number=%s" % current_build_number,
     ]
+    if notify:
+        parts.append("--notify")
     return [
         {"wait": "~", "continue_on_failure": "true"},
         create_step(
@@ -2748,6 +2763,7 @@ def main(argv=None):
     bazel_downstream_pipeline.add_argument(
         "--test_disabled_projects", type=bool, nargs="?", const=True
     )
+    bazel_downstream_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
 
     project_pipeline = subparsers.add_parser("project_pipeline")
     project_pipeline.add_argument("--project_name", type=str)
@@ -2757,6 +2773,7 @@ def main(argv=None):
     project_pipeline.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
     project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
     project_pipeline.add_argument("--incompatible_flag", type=str, action="append")
+    project_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
 
     runner = subparsers.add_parser("runner")
     runner.add_argument("--task", action="store", type=str, default="")
@@ -2808,6 +2825,7 @@ def main(argv=None):
                 file_config=args.file_config,
                 test_incompatible_flags=args.test_incompatible_flags,
                 test_disabled_projects=args.test_disabled_projects,
+                notify=args.notify,
             )
         elif args.subparsers_name == "project_pipeline":
             configs = fetch_configs(args.http_config, args.file_config)
@@ -2820,6 +2838,7 @@ def main(argv=None):
                 monitor_flaky_tests=args.monitor_flaky_tests,
                 use_but=args.use_but,
                 incompatible_flags=args.incompatible_flag,
+                notify=args.notify,
             )
         elif args.subparsers_name == "runner":
             configs = fetch_configs(args.http_config, args.file_config)