From 87cd809513a782efd698ef759744e9d4ce3ea3d6 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 14:25:38 +0100 Subject: [PATCH 1/7] Regression tests: require manual approval for certified connectors --- .../approve-regression-tests-command.yml | 114 ++++++++++++++++++ .github/workflows/slash-commands.yml | 1 + airbyte-ci/connectors/pipelines/README.md | 1 + .../airbyte_ci/connectors/test/commands.py | 16 ++- .../connectors/pipelines/pyproject.toml | 2 +- 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/approve-regression-tests-command.yml diff --git a/.github/workflows/approve-regression-tests-command.yml b/.github/workflows/approve-regression-tests-command.yml new file mode 100644 index 000000000000..831d54c27d3b --- /dev/null +++ b/.github/workflows/approve-regression-tests-command.yml @@ -0,0 +1,114 @@ +name: Approve Regression Tests +permissions: + pull-requests: write +on: + workflow_dispatch: + inputs: + pr: + description: "Pull request number. Used to pull the proper branch ref, including on forks." + type: number + required: false + comment-id: + description: "Optional. The comment-id of the slash command. Used to update the comment with the status." + required: false + + # These must be declared, but they are unused and ignored. + # TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these. + repo: + description: "Repo (Ignored)" + required: false + default: "airbytehq/airbyte" + gitref: + description: "Ref (Ignored)" + required: false + +run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}" +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.pr }} + # Cancel any previous runs on the same branch if they are still in progress + cancel-in-progress: true + +jobs: + approve-regression-tests: + name: "Approve Regression Tests" + runs-on: ubuntu-latest + steps: + - name: Get job variables + id: job-vars + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}) + echo "PR_JSON: $PR_JSON" + echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT + BRANCH=$(echo "$PR_JSON" | jq -r .head.ref) + echo "branch=$BRANCH" >> $GITHUB_OUTPUT + echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT + LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha) + echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT + + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ steps.job-vars.outputs.repo }} + ref: ${{ steps.job-vars.outputs.branch }} + fetch-depth: 1 + # Important that token is a PAT so that CI checks are triggered again. + # Without this we would be forever waiting on required checks to pass. + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + + - name: Append comment with job run link + # If comment-id is not provided, this will create a new + # comment with the job run link. + id: first-comment-action + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.inputs.comment-id }} + issue-number: ${{ github.event.inputs.pr }} + body: | + + > [Check job output.][1] + + [1]: ${{ steps.job-vars.outputs.run-url }} + + - name: Approve regression tests + id: approve + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "approving ...." + response=$(curl --write-out '%{http_code}' --silent --output /dev/null \ + --request POST \ + --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \ + --header 'authorization: Bearer ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}' \ + --header 'content-type: application/json' \ + --data '{ + "state": "success", + "context": "Regression tests manual approval", + "target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests" + }') + if [ $response -ne 201 ]; then + echo "Failed to approve regression tests. HTTP status code: $response" + exit 1 + else + echo "Regression tests approved." + fi + + - name: Append success comment + uses: peter-evans/create-or-update-comment@v4 + if: success() + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: "+1" + body: | + > ✅ Approving regression tests + + - name: Append failure comment + uses: peter-evans/create-or-update-comment@v4 + if: failure() + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: confused + body: | + > ❌ Job failed diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index afef1b4689d5..3ab57323c6c0 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -27,6 +27,7 @@ jobs: test-performance publish-java-cdk connector-performance + approve-regression-tests static-args: | repo=${{ steps.getref.outputs.repo }} gitref=${{ steps.getref.outputs.ref }} diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index f14bdbd633e9..e1006331a8f2 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -767,6 +767,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.24.0 | [#41627](https://github.com/airbytehq/airbyte/pull/41627) | Require manual regression test approval for certified connectors | | 4.23.1 | [#41541](https://github.com/airbytehq/airbyte/pull/41541) | Add support for submodule use-case. | | 4.23.0 | [#39906](https://github.com/airbytehq/airbyte/pull/39906) | Add manifest only build pipeline | | 4.22.0 | [#41623](https://github.com/airbytehq/airbyte/pull/41623) | Make `airbyte-ci` run on private forks. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 738011182035..68861b394747 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -14,16 +14,17 @@ from pipelines.airbyte_ci.connectors.test.steps.common import LiveTests from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, ContextState +from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, CIContext, ContextState from pipelines.helpers.execution import argument_parsing from pipelines.helpers.execution.run_steps import RunStepOptions -from pipelines.helpers.github import update_global_commit_status_check_for_tests +from pipelines.helpers.github import update_commit_status_check, update_global_commit_status_check_for_tests from pipelines.helpers.utils import fail_if_missing_docker_hub_creds from pipelines.models.secrets import GSMSecretStore from pipelines.models.steps import STEP_PARAMS GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests" GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests" +REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT = "Regression tests manual approval" TESTS_SKIPPED_BY_DEFAULT = [ CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS, ] @@ -124,6 +125,17 @@ async def test( if ctx.obj["selected_connectors_with_modified_files"]: update_global_commit_status_check_for_tests(ctx.obj, "pending") + if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors"]]): + update_commit_status_check( + ctx.obj["git_revision"], + "failure", + ctx.obj["gha_workflow_run_url"], + description="Check if regression tests have been manually approved", + context=REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT, + is_optional=False, + should_send=ctx.obj.get("ci_context") == CIContext.PULL_REQUEST, + logger=main_logger, + ) else: main_logger.warn("No connector were selected for testing.") update_global_commit_status_check_for_tests(ctx.obj, "success") diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 95915608d5a6..897b8ac31cc9 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.23.1" +version = "4.24.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 076e37284d96f35f9250cafa8911079ad4664d3a Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 11:19:51 -0400 Subject: [PATCH 2/7] Update .github/workflows/approve-regression-tests-command.yml Co-authored-by: Augustin --- .github/workflows/approve-regression-tests-command.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/approve-regression-tests-command.yml b/.github/workflows/approve-regression-tests-command.yml index 831d54c27d3b..a6c7dc9c2ce9 100644 --- a/.github/workflows/approve-regression-tests-command.yml +++ b/.github/workflows/approve-regression-tests-command.yml @@ -1,6 +1,7 @@ name: Approve Regression Tests permissions: pull-requests: write + status: write # to allow the GITHUB_TOKEN to post a commit status on: workflow_dispatch: inputs: From 7bfdb516cfd93b31dc8d71691211343ac7d3cb36 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 11:20:36 -0400 Subject: [PATCH 3/7] Update .github/workflows/approve-regression-tests-command.yml Co-authored-by: Augustin --- .github/workflows/approve-regression-tests-command.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/approve-regression-tests-command.yml b/.github/workflows/approve-regression-tests-command.yml index a6c7dc9c2ce9..f13d6df19375 100644 --- a/.github/workflows/approve-regression-tests-command.yml +++ b/.github/workflows/approve-regression-tests-command.yml @@ -49,16 +49,6 @@ jobs: LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha) echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ steps.job-vars.outputs.repo }} - ref: ${{ steps.job-vars.outputs.branch }} - fetch-depth: 1 - # Important that token is a PAT so that CI checks are triggered again. - # Without this we would be forever waiting on required checks to pass. - token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - - name: Append comment with job run link # If comment-id is not provided, this will create a new # comment with the job run link. From 819157202e880d8e5fd59faf4c69f874b6a51870 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 11:20:43 -0400 Subject: [PATCH 4/7] Update .github/workflows/approve-regression-tests-command.yml Co-authored-by: Augustin --- .github/workflows/approve-regression-tests-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/approve-regression-tests-command.yml b/.github/workflows/approve-regression-tests-command.yml index f13d6df19375..0fb2fb3deae7 100644 --- a/.github/workflows/approve-regression-tests-command.yml +++ b/.github/workflows/approve-regression-tests-command.yml @@ -72,7 +72,7 @@ jobs: response=$(curl --write-out '%{http_code}' --silent --output /dev/null \ --request POST \ --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \ - --header 'authorization: Bearer ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}' \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ --header 'content-type: application/json' \ --data '{ "state": "success", From d396ee05e7b1a819931c972b669f6b99564b5643 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 11:22:49 -0400 Subject: [PATCH 5/7] Update .github/workflows/approve-regression-tests-command.yml Co-authored-by: Augustin --- .github/workflows/approve-regression-tests-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/approve-regression-tests-command.yml b/.github/workflows/approve-regression-tests-command.yml index 0fb2fb3deae7..223519fd782f 100644 --- a/.github/workflows/approve-regression-tests-command.yml +++ b/.github/workflows/approve-regression-tests-command.yml @@ -102,4 +102,4 @@ jobs: comment-id: ${{ steps.first-comment-action.outputs.comment-id }} reactions: confused body: | - > ❌ Job failed + > ❌ Regression test approval failed From 754e5816c3b4aaf4abf8871e271b199c8ea3f3ee Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 16:29:06 +0100 Subject: [PATCH 6/7] cr comments --- .../approve-regression-tests-command.yml | 8 +------ .../airbyte_ci/connectors/test/commands.py | 17 ++++---------- .../connectors/pipelines/pipelines/hacks.py | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/approve-regression-tests-command.yml b/.github/workflows/approve-regression-tests-command.yml index 223519fd782f..55442731f71d 100644 --- a/.github/workflows/approve-regression-tests-command.yml +++ b/.github/workflows/approve-regression-tests-command.yml @@ -24,10 +24,6 @@ on: required: false run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}" -concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.pr }} - # Cancel any previous runs on the same branch if they are still in progress - cancel-in-progress: true jobs: approve-regression-tests: @@ -65,8 +61,6 @@ jobs: - name: Approve regression tests id: approve - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "approving ...." response=$(curl --write-out '%{http_code}' --silent --output /dev/null \ @@ -85,7 +79,7 @@ jobs: else echo "Regression tests approved." fi - + - name: Append success comment uses: peter-evans/create-or-update-comment@v4 if: success() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 68861b394747..03ebf989556a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -14,10 +14,11 @@ from pipelines.airbyte_ci.connectors.test.steps.common import LiveTests from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, CIContext, ContextState +from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, ContextState +from pipelines.hacks import do_regression_test_status_check_maybe from pipelines.helpers.execution import argument_parsing from pipelines.helpers.execution.run_steps import RunStepOptions -from pipelines.helpers.github import update_commit_status_check, update_global_commit_status_check_for_tests +from pipelines.helpers.github import update_global_commit_status_check_for_tests from pipelines.helpers.utils import fail_if_missing_docker_hub_creds from pipelines.models.secrets import GSMSecretStore from pipelines.models.steps import STEP_PARAMS @@ -125,17 +126,7 @@ async def test( if ctx.obj["selected_connectors_with_modified_files"]: update_global_commit_status_check_for_tests(ctx.obj, "pending") - if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors"]]): - update_commit_status_check( - ctx.obj["git_revision"], - "failure", - ctx.obj["gha_workflow_run_url"], - description="Check if regression tests have been manually approved", - context=REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT, - is_optional=False, - should_send=ctx.obj.get("ci_context") == CIContext.PULL_REQUEST, - logger=main_logger, - ) + do_regression_test_status_check_maybe(ctx, REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT, main_logger) else: main_logger.warn("No connector were selected for testing.") update_global_commit_status_check_for_tests(ctx.obj, "success") diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index e11cc5664b23..d9a8ac6aef94 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -6,9 +6,12 @@ from __future__ import annotations +from logging import Logger from typing import TYPE_CHECKING, Callable, List +import asyncclick as click from pipelines import consts +from pipelines.helpers.github import update_commit_status_check if TYPE_CHECKING: from dagger import Container @@ -70,3 +73,22 @@ def never_fail_exec_inner(container: Container) -> Container: return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True) return never_fail_exec_inner + + +def do_regression_test_status_check_maybe(ctx: click.Context, status_check_name: str, logger: Logger) -> None: + """ + Emit a failing status check that requires a manual override, via a /-command. + + Only required for certified connectors. + """ + if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors"]]): + update_commit_status_check( + ctx.obj["git_revision"], + "failure", + ctx.obj["gha_workflow_run_url"], + description="Check if regression tests have been manually approved", + context=status_check_name, + is_optional=False, + should_send=ctx.obj.get("ci_context") == consts.CIContext.PULL_REQUEST, + logger=logger, + ) From 262bf4a8324a6ce92af549ec441af36971fd8a1e Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 11 Jul 2024 17:16:33 +0100 Subject: [PATCH 7/7] fix --- airbyte-ci/connectors/pipelines/pipelines/hacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index d9a8ac6aef94..0fdece86c009 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -81,7 +81,7 @@ def do_regression_test_status_check_maybe(ctx: click.Context, status_check_name: Only required for certified connectors. """ - if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors"]]): + if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors_with_modified_files"]]): update_commit_status_check( ctx.obj["git_revision"], "failure",