From 092dac6482acde0fd800607fa6c090f1abed3dda Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 30 Apr 2024 15:52:26 +0200 Subject: [PATCH] airbyte-ci: custom CI status checks --- .github/workflows/community_ci.yml | 62 +++++++++++++++++-- airbyte-ci/connectors/pipelines/README.md | 1 + .../airbyte_ci/connectors/test/commands.py | 22 ++++++- .../pipelines/pipelines/helpers/github.py | 6 +- .../connectors/pipelines/pyproject.toml | 2 +- 5 files changed, 81 insertions(+), 12 deletions(-) diff --git a/.github/workflows/community_ci.yml b/.github/workflows/community_ci.yml index 1a8ba9ee88ff..2d782b53d43a 100644 --- a/.github/workflows/community_ci.yml +++ b/.github/workflows/community_ci.yml @@ -42,12 +42,13 @@ jobs: # This will sync the .github folder of the main repo with the fork # This allows us to use up to date actions from the main repo - - name: Pull .github folder from main repository + - name: Pull .github folder and internal packages from main repository id: pull_github_folder run: | git remote add main https://github.com/airbytehq/airbyte.git git fetch main ${MAIN_BRANCH_NAME} git checkout main/${MAIN_BRANCH_NAME} -- .github + git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci - name: Run airbyte-ci format check all # This path refers to the fork .github folder. @@ -58,8 +59,56 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} subcommand: "format check all" is_fork: "true" - connectors_test: - name: Run connectors tests on fork + connectors_early_ci: + name: Run connectors early CI on fork + if: github.event.pull_request.head.repo.fork == true + environment: community-ci-auto + runs-on: community-tooling-test-small + timeout-minutes: 10 + env: + MAIN_BRANCH_NAME: "master" + permissions: + statuses: write + steps: + # This checkouts a fork which can contain untrusted code + # It's deemed safe as the static checks are not executing any checked out code + - name: Checkout fork + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 1 + + # This will sync the .github folder of the main repo with the fork + # This allows us to use up to date actions and internal packages logic from the main repo + # It will also prevent forks from changing CI logic + - name: Pull .github folder from main repository + id: pull_github_folder + run: | + git remote add main https://github.com/airbytehq/airbyte.git + git fetch main ${MAIN_BRANCH_NAME} + git checkout main/${MAIN_BRANCH_NAME} -- .github + git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci + + - name: Run airbyte-ci static checks and version increment checks on modified connectors + # This path refers to the fork .github folder. + # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step + uses: ./.github/actions/run-airbyte-ci + with: + context: "pull_request" + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + subcommand: "connectors --modified test --only-step=qa_checks --only-step=version_inc_check --global-status-check-context='Connectors early CI checks' --global-status-check-description='Running early CI checks on connectors'" + is_fork: "true" + - name: Upload pipeline reports + id: upload-artifact + uses: actions/upload-artifact@v4 + with: + name: early-ci-pipeline-reports + path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html + retention-days: 7 + + connectors_full_ci: + name: Run all connectors tests on fork if: github.event.pull_request.head.repo.fork == true # Deployment of jobs on the community-ci environment requires manual approval # This is something we set up in the GitHub environment settings: @@ -83,14 +132,15 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 1 - # This will sync the .github folder of the main repo with the fork - # This allows us to use up to date actions from the main repo - - name: Pull .github folder from main repository + # This will sync the .github folder of the main repo with the fork + # This allows us to use up to date actions and internal packages logic from the main repo + # It will also prevent forks from changing CI logic - name: Pull .github folder from main repository id: pull_github_folder run: | git remote add main https://github.com/airbytehq/airbyte.git git fetch main ${MAIN_BRANCH_NAME} git checkout main/${MAIN_BRANCH_NAME} -- .github + git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci - name: Run airbyte-ci connectors test # This path refers to the fork .github folder. diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 6803dfb429cf..257815d9b539 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -676,6 +676,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | |---------| ---------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------| +| 4.12.0 | [#37690](https://github.com/airbytehq/airbyte/pull/37690) | Pass custom CI status name in `connectors test` | | 4.11.0 | [#37641](https://github.com/airbytehq/airbyte/pull/37641) | Updates to run regression tests in GitHub Actions. | | 4.10.5 | [#37641](https://github.com/airbytehq/airbyte/pull/37641) | Reintroduce changes from 4.10.0 with a fix. | | 4.10.4 | [#37641](https://github.com/airbytehq/airbyte/pull/37641) | Temporarily revert changes from version 4.10.0 | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 808c38ead9b5..e6885d6b4365 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -21,6 +21,9 @@ from pipelines.helpers.utils import fail_if_missing_docker_hub_creds from pipelines.models.steps import STEP_PARAMS +GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests" +GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests" + @click.command( cls=DaggerPipelineCommand, @@ -67,6 +70,18 @@ type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]), help="Only run specific step by name. Can be used multiple times to keep multiple steps.", ) +@click.option( + "--global-status-check-context", + "global_status_check_context", + help="The context of the global status check which will be sent to GitHub status API.", + default=GITHUB_GLOBAL_CONTEXT_FOR_TESTS, +) +@click.option( + "--global-status-check-description", + "global_status_check_description", + help="The description of the global status check which will be sent to GitHub status API.", + default=GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS, +) @click.argument( "extra_params", nargs=-1, type=click.UNPROCESSED, callback=argument_parsing.build_extra_params_mapping(CONNECTOR_TEST_STEP_ID) ) @@ -78,6 +93,8 @@ async def test( concurrent_cat: bool, skip_steps: List[str], only_steps: List[str], + global_status_check_context: str, + global_status_check_description: str, extra_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS], ) -> bool: """Runs a test pipeline for the selected connectors. @@ -85,6 +102,9 @@ async def test( Args: ctx (click.Context): The click context. """ + ctx.obj["global_status_check_context"] = global_status_check_context + ctx.obj["global_status_check_description"] = global_status_check_description + if only_steps and skip_steps: raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.") if not only_steps: @@ -109,7 +129,7 @@ async def test( connectors_tests_contexts = [ ConnectorContext( - pipeline_name=f"Testing connector {connector.technical_name}", + pipeline_name=f"{global_status_check_context} on {connector.technical_name}", connector=connector, is_local=ctx.obj["is_local"], git_branch=ctx.obj["git_branch"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py index 867c0fa896b7..999cbed533c9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py @@ -19,8 +19,6 @@ from github import Github, PullRequest AIRBYTE_GITHUB_REPO = "airbytehq/airbyte" -GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests" -GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests" def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> None: @@ -104,8 +102,8 @@ def update_global_commit_status_check_for_tests(click_context: dict, github_stat click_context["git_revision"], github_state, click_context["gha_workflow_run_url"], - GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS, - GITHUB_GLOBAL_CONTEXT_FOR_TESTS, + click_context["global_status_check_description"], + click_context["global_status_check_context"], should_send=click_context.get("ci_context") == CIContext.PULL_REQUEST, logger=logger, ) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 188d07df113d..2163abd845ac 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.11.0" +version = "4.12.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]