From 182707bae302fc0f03f8181a0d6390ad52c5ec54 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Fri, 5 Apr 2024 13:53:55 +0200 Subject: [PATCH] [invoke] Add tasks to query CODEOWNERS files (#24112) * [invoke] Add tasks to use CODEOWNERS files * Add unit tests * Apply suggestions from code review Co-authored-by: Nicolas Schweitzer --------- Co-authored-by: Nicolas Schweitzer --- tasks/__init__.py | 2 ++ tasks/github_tasks.py | 2 +- tasks/libs/owners/__init__.py | 0 tasks/libs/owners/parsing.py | 14 ++++++++++ tasks/libs/pipeline/notifications.py | 8 +----- tasks/owners.py | 13 +++++++++ tasks/pipeline.py | 3 ++- .../golangci_lint_parser.py | 2 +- tasks/unit-tests/github_tasks_tests.py | 2 +- tasks/unit-tests/owners_lib_tests.py | 27 +++++++++++++++++++ tasks/unit-tests/testdata/jobowners.txt | 7 +++++ 11 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 tasks/libs/owners/__init__.py create mode 100644 tasks/libs/owners/parsing.py create mode 100644 tasks/owners.py create mode 100644 tasks/unit-tests/owners_lib_tests.py create mode 100644 tasks/unit-tests/testdata/jobowners.txt diff --git a/tasks/__init__.py b/tasks/__init__.py index 93040b0aad7fb..7a27933b2ad90 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -28,6 +28,7 @@ msi, new_e2e_tests, notify, + owners, package, pipeline, process_agent, @@ -155,6 +156,7 @@ ns.add_collection(kmt) ns.add_collection(diff) ns.add_collection(updater) +ns.add_collection(owners) ns.add_collection(modules) ns.configure( { diff --git a/tasks/github_tasks.py b/tasks/github_tasks.py index a5ab0a9398d59..b074b9aa65b4a 100644 --- a/tasks/github_tasks.py +++ b/tasks/github_tasks.py @@ -18,7 +18,7 @@ from tasks.libs.common.datadog_api import create_count, send_metrics from tasks.libs.common.junit_upload_core import repack_macos_junit_tar from tasks.libs.common.utils import DEFAULT_BRANCH, DEFAULT_INTEGRATIONS_CORE_BRANCH, get_git_pretty_ref -from tasks.libs.pipeline.notifications import read_owners +from tasks.libs.owners.parsing import read_owners from tasks.release import _get_release_json_value diff --git a/tasks/libs/owners/__init__.py b/tasks/libs/owners/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/libs/owners/parsing.py b/tasks/libs/owners/parsing.py new file mode 100644 index 0000000000000..a5122a445aa7a --- /dev/null +++ b/tasks/libs/owners/parsing.py @@ -0,0 +1,14 @@ +from typing import Any, List + + +def read_owners(owners_file: str) -> Any: + from codeowners import CodeOwners + + with open(owners_file, 'r') as f: + return CodeOwners(f.read()) + + +def search_owners(search: str, owners_file: str) -> List[str]: + parsed_owners = read_owners(owners_file) + # owners.of returns a list in the form: [('TEAM', '@DataDog/agent-build-and-releases')] + return [owner[1] for owner in parsed_owners.of(search)] diff --git a/tasks/libs/pipeline/notifications.py b/tasks/libs/pipeline/notifications.py index d7247ee7cc676..03d4ba7f12d62 100644 --- a/tasks/libs/pipeline/notifications.py +++ b/tasks/libs/pipeline/notifications.py @@ -9,6 +9,7 @@ import yaml from tasks.libs.ciproviders.gitlab import Gitlab, get_gitlab_token +from tasks.libs.owners.parsing import read_owners from tasks.libs.types.types import FailedJobs, Test @@ -32,13 +33,6 @@ def load_and_validate(file_name: str, default_placeholder: str, default_value: s GITHUB_JIRA_MAP = load_and_validate("github_jira_map.yaml", "DEFAULT_JIRA_PROJECT", DEFAULT_JIRA_PROJECT) -def read_owners(owners_file): - from codeowners import CodeOwners - - with open(owners_file, 'r') as f: - return CodeOwners(f.read()) - - def check_for_missing_owners_slack_and_jira(print_missing_teams=True, owners_file=".github/CODEOWNERS"): owners = read_owners(owners_file) error = False diff --git a/tasks/owners.py b/tasks/owners.py new file mode 100644 index 0000000000000..d09c13f948d27 --- /dev/null +++ b/tasks/owners.py @@ -0,0 +1,13 @@ +from invoke import task + +from tasks.libs.owners.parsing import search_owners + + +@task +def find_jobowners(_, job, owners_file=".gitlab/JOBOWNERS"): + print(", ".join(search_owners(job, owners_file))) + + +@task +def find_codeowners(_, path, owners_file=".github/CODEOWNERS"): + print(", ".join(search_owners(path, owners_file))) diff --git a/tasks/pipeline.py b/tasks/pipeline.py index d005c07c79a01..9a9478e7f9e8e 100644 --- a/tasks/pipeline.py +++ b/tasks/pipeline.py @@ -21,7 +21,8 @@ nightly_entry_for, release_entry_for, ) -from tasks.libs.pipeline.notifications import read_owners, send_slack_message +from tasks.libs.owners.parsing import read_owners +from tasks.libs.pipeline.notifications import send_slack_message from tasks.libs.pipeline.tools import ( FilteredOutException, cancel_pipelines_with_confirmation, diff --git a/tasks/show_linters_issues/golangci_lint_parser.py b/tasks/show_linters_issues/golangci_lint_parser.py index 6a181190aecd2..1aa22f20f4c8b 100644 --- a/tasks/show_linters_issues/golangci_lint_parser.py +++ b/tasks/show_linters_issues/golangci_lint_parser.py @@ -7,7 +7,7 @@ from collections import defaultdict from typing import Dict -from tasks.libs.pipeline.notifications import read_owners +from tasks.libs.owners.parsing import read_owners # Example lint message # "pointer.go:6:1: package-comments: should have a package comment (revive)" diff --git a/tasks/unit-tests/github_tasks_tests.py b/tasks/unit-tests/github_tasks_tests.py index 85ba43f392eb5..a38494ed420de 100644 --- a/tasks/unit-tests/github_tasks_tests.py +++ b/tasks/unit-tests/github_tasks_tests.py @@ -27,7 +27,7 @@ class TestAssignTeamLabelMock(unittest.TestCase): CODEOWNERS_FILE = './tasks/unit-tests/testdata/codeowners.txt' def make_test(self, changed_files, expected_labels, pr_labels=None, possible_labels=None): - from tasks.libs.pipeline.notifications import read_owners + from tasks.libs.owners.parsing import read_owners possible_labels = possible_labels or ['team/agent-platform', 'team/documentation', 'team/agent-security'] diff --git a/tasks/unit-tests/owners_lib_tests.py b/tasks/unit-tests/owners_lib_tests.py new file mode 100644 index 0000000000000..c689c479d800f --- /dev/null +++ b/tasks/unit-tests/owners_lib_tests.py @@ -0,0 +1,27 @@ +import unittest + +from tasks.libs.owners.parsing import search_owners + + +class TestSearchCodeOwners(unittest.TestCase): + CODEOWNERS_FILE = './tasks/unit-tests/testdata/codeowners.txt' + JOBOWNERS_FILE = './tasks/unit-tests/testdata/jobowners.txt' + + def test_search_codeowners(self): + self.assertListEqual(search_owners("no_owners/file", self.CODEOWNERS_FILE), []) + self.assertListEqual(search_owners(".dotfile", self.CODEOWNERS_FILE), ["@DataDog/agent-platform"]) + self.assertListEqual( + search_owners("doc.md", self.CODEOWNERS_FILE), ["@DataDog/agent-platform", "@DataDog/documentation"] + ) + self.assertListEqual(search_owners(".gitlab/security.yml", self.CODEOWNERS_FILE), ["@DataDog/agent-security"]) + + def test_search_jobowners(self): + self.assertListEqual(search_owners("default_job", self.JOBOWNERS_FILE), ["@DataDog/agent-ci-experience"]) + self.assertListEqual(search_owners("tests_default", self.JOBOWNERS_FILE), ["@DataDog/multiple"]) + self.assertListEqual(search_owners("tests_ebpf_x64", self.JOBOWNERS_FILE), ["@DataDog/ebpf-platform"]) + self.assertListEqual( + search_owners("security_go_generate_check", self.JOBOWNERS_FILE), ["@DataDog/agent-security"] + ) + self.assertListEqual( + search_owners("security_go_generate_checks", self.JOBOWNERS_FILE), ["@DataDog/agent-ci-experience"] + ) diff --git a/tasks/unit-tests/testdata/jobowners.txt b/tasks/unit-tests/testdata/jobowners.txt new file mode 100644 index 0000000000000..a1a9ae83a8c8d --- /dev/null +++ b/tasks/unit-tests/testdata/jobowners.txt @@ -0,0 +1,7 @@ +# Extract of the .gitab/JOBOWNERS file, for testing purposes + +* @DataDog/agent-ci-experience + +tests_* @DataDog/multiple +tests_ebpf* @DataDog/ebpf-platform +security_go_generate_check @DataDog/agent-security