From e77959cb9214be14bed1600908f77070a4926861 Mon Sep 17 00:00:00 2001 From: Augustin Date: Fri, 20 Oct 2023 12:14:37 +0200 Subject: [PATCH] [airbyte-ci]: new check on python certified connector to validate their base image use (#30527) Co-authored-by: alafanechere --- airbyte-ci/connectors/pipelines/README.md | 1 + .../connectors/test/steps/common.py | 26 +++++++++++++++++++ .../test/steps/python_connectors.py | 3 ++- .../connectors/pipelines/pyproject.toml | 2 +- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 186df98e1fd4..067bfa3fb569 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -398,6 +398,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | |---------| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.2.0 | [#30527](https://github.com/airbytehq/airbyte/pull/30527) | Add a new check for python connectors to make sure certified connectors use our base image. | 2.1.1 | [#31488](https://github.com/airbytehq/airbyte/pull/31488) | Improve `airbyte-ci` start time with Click Lazy load | | 2.1.0 | [#31412](https://github.com/airbytehq/airbyte/pull/31412) | Run airbyte-ci from any where in airbyte project | | 2.0.4 | [#31487](https://github.com/airbytehq/airbyte/pull/31487) | Allow for third party connector selections | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 309fed265a73..48fe917778b1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -285,3 +285,29 @@ async def _build_connector_acceptance_test(self, connector_under_test_image_tar: ) return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) + + +class CheckBaseImageIsUsed(Step): + title = "Check our base image is used" + + async def _run(self, *args, **kwargs) -> StepResult: + is_certified = self.context.connector.metadata.get("supportLevel") == "certified" + if not is_certified: + self.skip("Connector is not certified, it does not require the use of our base image.") + + is_using_base_image = self.context.connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is not None + migration_hint = f"Please run 'airbyte-ci connectors --name={self.context.connector.technical_name} migrate_to_base_image {self.context.pull_request.number}' and commit the changes." + if not is_using_base_image: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Connector is certified but does not use our base image. {migration_hint}", + ) + has_dockerfile = "Dockerfile" in await (await self.context.get_connector_dir(include="Dockerfile")).entries() + if has_dockerfile: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Connector is certified but is still using a Dockerfile. {migration_hint}", + ) + return StepResult(self, StepStatus.SUCCESS, stdout="Connector is certified and uses our base image.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 640aa4ba8ae5..783e0baf8062 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -13,7 +13,7 @@ from dagger import Container, File from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests +from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, CheckBaseImageIsUsed from pipelines.consts import LOCAL_BUILD_PLATFORM, PYPROJECT_TOML_FILE_PATH from pipelines.dagger.actions import secrets from pipelines.helpers.utils import export_container_to_tarball @@ -228,6 +228,7 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]: tasks = [ task_group.soonify(IntegrationTests(context).run)(connector_container), task_group.soonify(AcceptanceTests(context).run)(connector_image_tar_file), + task_group.soonify(CheckBaseImageIsUsed(context).run)(), ] return step_results + [task.value for task in tasks] diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 84d8dabca86d..5b666ca3924c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.1.1" +version = "2.2.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]