diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index 5a8b90291e371..43df96ede1f61 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -76,7 +76,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/connector_ops" + subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'" airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} @@ -91,7 +91,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/pipelines" + subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests'" airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} @@ -106,7 +106,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/base_images" + subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'" airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} @@ -115,7 +115,7 @@ jobs: if: steps.changes.outputs.metadata_lib_any_changed == 'true' uses: ./.github/actions/run-dagger-pipeline with: - subcommand: "test airbyte-ci/connectors/metadata_service/lib/" + subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'" context: "pull_request" github_token: ${{ secrets.GITHUB_TOKEN }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} @@ -128,7 +128,7 @@ jobs: if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true' uses: ./.github/actions/run-dagger-pipeline with: - subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/" + subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'" context: "pull_request" github_token: ${{ secrets.GITHUB_TOKEN }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 24893e6f3b6c5..8e4bd299a777d 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -506,17 +506,23 @@ This command runs the Python tests for a airbyte-ci poetry package. | Option | Required | Default | Mapped environment variable | Description | | ------------------ | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------------ | -| `--test-directory` | False | tests | | The path to the directory on which pytest should discover tests, relative to the poetry package. | +| `-c/--poetry-run-command` | True | None | | The command to run with `poetry run` | -#### Example +#### Examples +You can pass multiple `-c/--poetry-run-command` options to run multiple commands. + +E.G.: running `pytest` and `mypy`: +`airbyte-ci test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines'` -`airbyte-ci test airbyte-ci/connectors/pipelines --test-directory=tests` -`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests` +E.G.: running `pytest` on a specific test folder: +`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --poetry-run-command='pytest tests/unit_tests'` ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.13.0 | [#33784](https://github.com/airbytehq/airbyte/pull/33784) | Make `airbyte-ci test` able to run any poetry command | +| 2.12.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Add upgrade CDK command | | 2.11.0 | [#32188](https://github.com/airbytehq/airbyte/pull/32188) | Add -x option to connector test to allow for skipping steps | | 2.10.12 | [#33419](https://github.com/airbytehq/airbyte/pull/33419) | Make ClickPipelineContext handle dagger logging. | | 2.10.11 | [#33497](https://github.com/airbytehq/airbyte/pull/33497) | Consider nested .gitignore rules in format. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 4d51f3ad34b11..c3d002a017c78 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -1,19 +1,48 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations import logging +from pathlib import Path +from typing import TYPE_CHECKING import asyncclick as click +import asyncer from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +if TYPE_CHECKING: + from typing import List, Tuple + + import dagger + +## HELPERS +async def run_poetry_command(container: dagger.Container, command: str) -> Tuple[str, str]: + """Run a poetry command in a container and return the stdout and stderr. + + Args: + container (dagger.Container): The container to run the command in. + command (str): The command to run. + + Returns: + Tuple[str, str]: The stdout and stderr of the command. + """ + container = container.with_exec(["poetry", "run", *command.split(" ")]) + return await container.stdout(), await container.stderr() + @click.command() @click.argument("poetry_package_path") -@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") +@click.option( + "-c", + "--poetry-run-command", + multiple=True, + help="The poetry run command to run.", + required=True, +) @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs @@ -24,7 +53,10 @@ async def test(pipeline_context: ClickPipelineContext): pipeline_context (ClickPipelineContext): The context object. """ poetry_package_path = pipeline_context.params["poetry_package_path"] - test_directory = pipeline_context.params["test_directory"] + if not Path(f"{poetry_package_path}/pyproject.toml").exists(): + raise click.UsageError(f"Could not find pyproject.toml in {poetry_package_path}") + + commands_to_run: List[str] = pipeline_context.params["poetry_run_command"] logger = logging.getLogger(f"{poetry_package_path}.tests") logger.info(f"Running tests for {poetry_package_path}") @@ -47,7 +79,7 @@ async def test(pipeline_context: ClickPipelineContext): pipeline_name = f"Unit tests for {poetry_package_path}" dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name) - pytest_container = await ( + test_container = await ( dagger_client.container() .from_("python:3.10.12") .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") @@ -73,10 +105,20 @@ async def test(pipeline_context: ClickPipelineContext): ), ) .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) + .with_exec(["poetry", "install", "--with=dev"]) .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) .with_env_variable("CI", str(pipeline_context.params["is_ci"])) - .with_exec(["poetry", "run", "pytest", test_directory]) + .with_workdir(f"/airbyte/{poetry_package_path}") ) - await pytest_container + soon_command_executions_results = [] + async with asyncer.create_task_group() as poetry_commands_task_group: + for command in commands_to_run: + logger.info(f"Running command: {command}") + soon_command_execution_result = poetry_commands_task_group.soonify(run_poetry_command)(test_container, command) + soon_command_executions_results.append(soon_command_execution_result) + + for result in soon_command_executions_results: + stdout, stderr = result.value + logger.info(stdout) + logger.error(stderr) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index a5b0aaf3d3cdf..854d8ef171896 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.12.0" +version = "2.13.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte <contact@airbyte.io>"]