Regression tests: require manual approval for certified connectors (#…

…41627)

Co-authored-by: Augustin <augustin@airbyte.io>

.github/workflows/approve-regression-tests-command.yml

@@ -0,0 +1,99 @@
+name: Approve Regression Tests
+permissions:
+  pull-requests: write
+  status: write # to allow the GITHUB_TOKEN to post a commit status
+on:
+  workflow_dispatch:
+    inputs:
+      pr:
+        description: "Pull request number. Used to pull the proper branch ref, including on forks."
+        type: number
+        required: false
+      comment-id:
+        description: "Optional. The comment-id of the slash command. Used to update the comment with the status."
+        required: false
+
+      # These must be declared, but they are unused and ignored.
+      # TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these.
+      repo:
+        description: "Repo (Ignored)"
+        required: false
+        default: "airbytehq/airbyte"
+      gitref:
+        description: "Ref (Ignored)"
+        required: false
+
+run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}"
+
+jobs:
+  approve-regression-tests:
+    name: "Approve Regression Tests"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get job variables
+        id: job-vars
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
+          echo "PR_JSON: $PR_JSON"
+          echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
+          BRANCH=$(echo "$PR_JSON" | jq -r .head.ref)
+          echo "branch=$BRANCH" >> $GITHUB_OUTPUT
+          echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
+          LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha)
+          echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT
+
+      - name: Append comment with job run link
+        # If comment-id is not provided, this will create a new
+        # comment with the job run link.
+        id: first-comment-action
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          issue-number: ${{ github.event.inputs.pr }}
+          body: |
+
+            > [Check job output.][1]
+
+            [1]: ${{ steps.job-vars.outputs.run-url }}
+
+      - name: Approve regression tests
+        id: approve
+        run: |
+          echo "approving ...."
+          response=$(curl --write-out '%{http_code}' --silent --output /dev/null \
+            --request POST \
+            --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \
+            --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+            --header 'content-type: application/json' \
+            --data '{
+              "state": "success",
+              "context": "Regression tests manual approval",
+              "target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests"
+            }')
+          if [ $response -ne 201 ]; then
+            echo "Failed to approve regression tests. HTTP status code: $response"
+            exit 1
+          else
+            echo "Regression tests approved."
+          fi
+
+      - name: Append success comment
+        uses: peter-evans/create-or-update-comment@v4
+        if: success()
+        with:
+          comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+          reactions: "+1"
+          body: |
+            > ✅ Approving regression tests
+
+      - name: Append failure comment
+        uses: peter-evans/create-or-update-comment@v4
+        if: failure()
+        with:
+          comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+          reactions: confused
+          body: |
+            > ❌ Regression test approval failed

.github/workflows/slash-commands.yml

@@ -27,6 +27,7 @@ jobs:
             test-performance
             publish-java-cdk
             connector-performance
+            approve-regression-tests
           static-args: |
             repo=${{ steps.getref.outputs.repo }}
             gitref=${{ steps.getref.outputs.ref }}

airbyte-ci/connectors/pipelines/README.md

@@ -767,6 +767,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.24.0  | [#41627](https://github.com/airbytehq/airbyte/pull/41627) | Require manual regression test approval for certified connectors                                                             |
 | 4.23.1  | [#41541](https://github.com/airbytehq/airbyte/pull/41541)  | Add support for submodule use-case.                                                                                          |
 | 4.23.0  | [#39906](https://github.com/airbytehq/airbyte/pull/39906)  | Add manifest only build pipeline                                                          |
 | 4.22.0  | [#41623](https://github.com/airbytehq/airbyte/pull/41623)  | Make `airbyte-ci` run on private forks.                                                                                      |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

@@ -15,6 +15,7 @@
 from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, ContextState
+from pipelines.hacks import do_regression_test_status_check_maybe
 from pipelines.helpers.execution import argument_parsing
 from pipelines.helpers.execution.run_steps import RunStepOptions
 from pipelines.helpers.github import update_global_commit_status_check_for_tests
@@ -24,6 +25,7 @@
 
 GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
 GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
+REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT = "Regression tests manual approval"
 TESTS_SKIPPED_BY_DEFAULT = [
     CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
 ]
@@ -124,6 +126,7 @@ async def test(
 
     if ctx.obj["selected_connectors_with_modified_files"]:
         update_global_commit_status_check_for_tests(ctx.obj, "pending")
+        do_regression_test_status_check_maybe(ctx, REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT, main_logger)
     else:
         main_logger.warn("No connector were selected for testing.")
         update_global_commit_status_check_for_tests(ctx.obj, "success")

airbyte-ci/connectors/pipelines/pipelines/hacks.py

@@ -6,9 +6,12 @@
 
 from __future__ import annotations
 
+from logging import Logger
 from typing import TYPE_CHECKING, Callable, List
 
+import asyncclick as click
 from pipelines import consts
+from pipelines.helpers.github import update_commit_status_check
 
 if TYPE_CHECKING:
     from dagger import Container
@@ -70,3 +73,22 @@ def never_fail_exec_inner(container: Container) -> Container:
         return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True)
 
     return never_fail_exec_inner
+
+
+def do_regression_test_status_check_maybe(ctx: click.Context, status_check_name: str, logger: Logger) -> None:
+    """
+    Emit a failing status check that requires a manual override, via a /-command.
+
+    Only required for certified connectors.
+    """
+    if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors_with_modified_files"]]):
+        update_commit_status_check(
+            ctx.obj["git_revision"],
+            "failure",
+            ctx.obj["gha_workflow_run_url"],
+            description="Check if regression tests have been manually approved",
+            context=status_check_name,
+            is_optional=False,
+            should_send=ctx.obj.get("ci_context") == consts.CIContext.PULL_REQUEST,
+            logger=logger,
+        )

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.23.1"
+version = "4.24.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]