Skip to content

Commit

Permalink
Regression tests: require manual approval for certified connectors
Browse files Browse the repository at this point in the history
  • Loading branch information
clnoll committed Jul 11, 2024
1 parent 59dd53e commit 49b1f29
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 3 deletions.
114 changes: 114 additions & 0 deletions .github/workflows/approve-regression-tests-command.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
name: Approve Regression Tests
permissions:
pull-requests: write
on:
workflow_dispatch:
inputs:
pr:
description: "Pull request number. Used to pull the proper branch ref, including on forks."
type: number
required: false
comment-id:
description: "Optional. The comment-id of the slash command. Used to update the comment with the status."
required: false

# These must be declared, but they are unused and ignored.
# TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these.
repo:
description: "Repo (Ignored)"
required: false
default: "airbytehq/airbyte"
gitref:
description: "Ref (Ignored)"
required: false

run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}"
concurrency:
group: ${{ github.workflow }}-${{ github.event.inputs.pr }}
# Cancel any previous runs on the same branch if they are still in progress
cancel-in-progress: true

jobs:
approve-regression-tests:
name: "Approve Regression Tests"
runs-on: ubuntu-latest
steps:
- name: Get job variables
id: job-vars
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
run: |
PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
echo "PR_JSON: $PR_JSON"
echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
BRANCH=$(echo "$PR_JSON" | jq -r .head.ref)
echo "branch=$BRANCH" >> $GITHUB_OUTPUT
echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha)
echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
repository: ${{ steps.job-vars.outputs.repo }}
ref: ${{ steps.job-vars.outputs.branch }}
fetch-depth: 1
# Important that token is a PAT so that CI checks are triggered again.
# Without this we would be forever waiting on required checks to pass.
token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}

- name: Append comment with job run link
# If comment-id is not provided, this will create a new
# comment with the job run link.
id: first-comment-action
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ github.event.inputs.comment-id }}
issue-number: ${{ github.event.inputs.pr }}
body: |
> [Check job output.][1]
[1]: ${{ steps.job-vars.outputs.run-url }}
- name: Approve regression tests
id: approve
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "approving ...."
response=$(curl --write-out '%{http_code}' --silent --output /dev/null \
--request POST \
--url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \
--header 'authorization: Bearer ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}' \
--header 'content-type: application/json' \
--data '{
"state": "success",
"context": "Regression tests manual approval",
"target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests"
}')
if [ $response -ne 201 ]; then
echo "Failed to approve regression tests. HTTP status code: $response"
exit 1
else
echo "Regression tests approved."
fi
- name: Append success comment
uses: peter-evans/create-or-update-comment@v4
if: success()
with:
comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
reactions: "+1"
body: |
> ✅ Approving regression tests
- name: Append failure comment
uses: peter-evans/create-or-update-comment@v4
if: failure()
with:
comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
reactions: confused
body: |
> ❌ Job failed
1 change: 1 addition & 0 deletions .github/workflows/slash-commands.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
test-performance
publish-java-cdk
connector-performance
approve-regression-tests
static-args: |
repo=${{ steps.getref.outputs.repo }}
gitref=${{ steps.getref.outputs.ref }}
Expand Down
1 change: 1 addition & 0 deletions airbyte-ci/connectors/pipelines/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:

| Version | PR | Description |
| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
| 4.22.0 | [#41627](https://github.com/airbytehq/airbyte/pull/41627) | Require manual regression test approval for certified connectors |
| 4.21.1 | [#41029](https://github.com/airbytehq/airbyte/pull/41029) | `up-to-date`: mount local docker config to `Syft` to pull private images and benefit from increased DockerHub rate limits. |
| 4.21.0 | [#40547](https://github.com/airbytehq/airbyte/pull/40547) | Make bump-version accept a `--pr-number` option. |
| 4.20.3 | [#40754](https://github.com/airbytehq/airbyte/pull/40754) | Accept and ignore additional args in `migrate-to-poetry` pipeline |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,17 @@
from pipelines.airbyte_ci.connectors.test.steps.common import LiveTests
from pipelines.cli.click_decorators import click_ci_requirements_option
from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, ContextState
from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, CIContext, ContextState
from pipelines.helpers.execution import argument_parsing
from pipelines.helpers.execution.run_steps import RunStepOptions
from pipelines.helpers.github import update_global_commit_status_check_for_tests
from pipelines.helpers.github import update_commit_status_check, update_global_commit_status_check_for_tests
from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
from pipelines.models.secrets import GSMSecretStore
from pipelines.models.steps import STEP_PARAMS

GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT = "Regression tests manual approval"
TESTS_SKIPPED_BY_DEFAULT = [
CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
]
Expand Down Expand Up @@ -124,6 +125,17 @@ async def test(

if ctx.obj["selected_connectors_with_modified_files"]:
update_global_commit_status_check_for_tests(ctx.obj, "pending")
if any([connector.support_level == "certified" for connector in ctx.obj["selected_connectors"]]):
update_commit_status_check(
ctx.obj["git_revision"],
"failure",
ctx.obj["gha_workflow_run_url"],
description="Check if regression tests have been manually approved",
context=REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT,
is_optional=False,
should_send=ctx.obj.get("ci_context") == CIContext.PULL_REQUEST,
logger=main_logger,
)
else:
main_logger.warn("No connector were selected for testing.")
update_global_commit_status_check_for_tests(ctx.obj, "success")
Expand Down
2 changes: 1 addition & 1 deletion airbyte-ci/connectors/pipelines/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "pipelines"
version = "4.21.1"
version = "4.22.0"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte <contact@airbyte.io>"]

Expand Down

0 comments on commit 49b1f29

Please sign in to comment.