community-ci: declare "early CI" (#37690)

.github/workflows/community_ci.yml

@@ -42,12 +42,13 @@ jobs:
 
       # This will sync the .github folder of the main repo with the fork
       # This allows us to use up to date actions from the main repo
-      - name: Pull .github folder from main repository
+      - name: Pull .github folder and internal packages from main repository
         id: pull_github_folder
         run: |
           git remote add main https://github.com/airbytehq/airbyte.git
           git fetch main ${MAIN_BRANCH_NAME}
           git checkout main/${MAIN_BRANCH_NAME} -- .github
+          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
 
       - name: Run airbyte-ci format check all
         # This path refers to the fork .github folder.
@@ -58,8 +59,56 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           subcommand: "format check all"
           is_fork: "true"
-  connectors_test:
-    name: Run connectors tests on fork
+  connectors_early_ci:
+    name: Run connectors early CI on fork
+    if: github.event.pull_request.head.repo.fork == true
+    environment: community-ci-auto
+    runs-on: community-tooling-test-small
+    timeout-minutes: 10
+    env:
+      MAIN_BRANCH_NAME: "master"
+    permissions:
+      statuses: write
+    steps:
+      # This checkouts a fork which can contain untrusted code
+      # It's deemed safe as the static checks are not executing any checked out code
+      - name: Checkout fork
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 1
+
+      # This will sync the .github folder of the main repo with the fork
+      # This allows us to use up to date actions and internal packages logic from the main repo
+      # It will also prevent forks from changing CI logic
+      - name: Pull .github folder from main repository
+        id: pull_github_folder
+        run: |
+          git remote add main https://github.com/airbytehq/airbyte.git
+          git fetch main ${MAIN_BRANCH_NAME}
+          git checkout main/${MAIN_BRANCH_NAME} -- .github
+          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
+
+      - name: Run airbyte-ci static checks and version increment checks on modified connectors
+        # This path refers to the fork .github folder.
+        # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "pull_request"
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          subcommand: "connectors --modified test --only-step=qa_checks --only-step=version_inc_check --global-status-check-context='Connectors early CI checks' --global-status-check-description='Running early CI checks on connectors'"
+          is_fork: "true"
+      - name: Upload pipeline reports
+        id: upload-artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: early-ci-pipeline-reports
+          path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html
+          retention-days: 7
+
+  connectors_full_ci:
+    name: Run all connectors tests on fork
     if: github.event.pull_request.head.repo.fork == true
     # Deployment of jobs on the community-ci environment requires manual approval
     # This is something we set up in the GitHub environment settings:
@@ -83,14 +132,15 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 1
 
-      # This will sync the .github folder of the main repo with the fork
-      # This allows us to use up to date actions from the main repo
-      - name: Pull .github folder from main repository
+        # This will sync the .github folder of the main repo with the fork
+        # This allows us to use up to date actions and internal packages logic from the main repo
+        # It will also prevent forks from changing CI logic      - name: Pull .github folder from main repository
         id: pull_github_folder
         run: |
           git remote add main https://github.com/airbytehq/airbyte.git
           git fetch main ${MAIN_BRANCH_NAME}
           git checkout main/${MAIN_BRANCH_NAME} -- .github
+          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
 
       - name: Run airbyte-ci connectors test
         # This path refers to the fork .github folder.

airbyte-ci/connectors/pipelines/README.md

@@ -676,6 +676,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 |---------| ---------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------|
+| 4.12.0  | [#37690](https://github.com/airbytehq/airbyte/pull/37690)  | Pass custom CI status name in `connectors test`                                                                               |
 | 4.11.0  | [#37641](https://github.com/airbytehq/airbyte/pull/37641)  | Updates to run regression tests in GitHub Actions.                                                                               |
 | 4.10.5  | [#37641](https://github.com/airbytehq/airbyte/pull/37641)  | Reintroduce changes from 4.10.0 with a fix.                                                                               |
 | 4.10.4  | [#37641](https://github.com/airbytehq/airbyte/pull/37641)  | Temporarily revert changes from version 4.10.0                                                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

@@ -21,6 +21,9 @@
 from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
 from pipelines.models.steps import STEP_PARAMS
 
+GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
+GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
+
 
 @click.command(
     cls=DaggerPipelineCommand,
@@ -67,6 +70,18 @@
     type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]),
     help="Only run specific step by name. Can be used multiple times to keep multiple steps.",
 )
+@click.option(
+    "--global-status-check-context",
+    "global_status_check_context",
+    help="The context of the global status check which will be sent to GitHub status API.",
+    default=GITHUB_GLOBAL_CONTEXT_FOR_TESTS,
+)
+@click.option(
+    "--global-status-check-description",
+    "global_status_check_description",
+    help="The description of the global status check which will be sent to GitHub status API.",
+    default=GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS,
+)
 @click.argument(
     "extra_params", nargs=-1, type=click.UNPROCESSED, callback=argument_parsing.build_extra_params_mapping(CONNECTOR_TEST_STEP_ID)
 )
@@ -78,13 +93,18 @@ async def test(
     concurrent_cat: bool,
     skip_steps: List[str],
     only_steps: List[str],
+    global_status_check_context: str,
+    global_status_check_description: str,
     extra_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS],
 ) -> bool:
     """Runs a test pipeline for the selected connectors.
 
     Args:
         ctx (click.Context): The click context.
     """
+    ctx.obj["global_status_check_context"] = global_status_check_context
+    ctx.obj["global_status_check_description"] = global_status_check_description
+
     if only_steps and skip_steps:
         raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.")
     if not only_steps:
@@ -109,7 +129,7 @@ async def test(
 
     connectors_tests_contexts = [
         ConnectorContext(
-            pipeline_name=f"Testing connector {connector.technical_name}",
+            pipeline_name=f"{global_status_check_context} on {connector.technical_name}",
             connector=connector,
             is_local=ctx.obj["is_local"],
             git_branch=ctx.obj["git_branch"],

airbyte-ci/connectors/pipelines/pipelines/helpers/github.py

@@ -19,8 +19,6 @@
 from github import Github, PullRequest
 
 AIRBYTE_GITHUB_REPO = "airbytehq/airbyte"
-GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
-GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
 
 
 def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> None:
@@ -104,8 +102,8 @@ def update_global_commit_status_check_for_tests(click_context: dict, github_stat
         click_context["git_revision"],
         github_state,
         click_context["gha_workflow_run_url"],
-        GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS,
-        GITHUB_GLOBAL_CONTEXT_FOR_TESTS,
+        click_context["global_status_check_description"],
+        click_context["global_status_check_context"],
         should_send=click_context.get("ci_context") == CIContext.PULL_REQUEST,
         logger=logger,
     )

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.11.0"
+version = "4.12.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]