From 72360cdf266c43be8590cf13d4e9c9ced6cdad19 Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 11 Oct 2023 15:53:31 +0200 Subject: [PATCH] [airbyte-ci] implement pre/post build hooks (#30526) --- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/builds/build_customization.py | 88 +++++++++++++++++++ .../pipelines/builds/python_connectors.py | 21 +++-- .../connectors/pipelines/pyproject.toml | 2 +- .../test_builds/dummy_build_customization.py | 35 ++++++++ .../test_builds/test_python_connectors.py | 54 ++++++++++-- .../connectors/source-zendesk-chat/Dockerfile | 20 ----- .../build_customization.py | 23 +++++ .../source-zendesk-chat/metadata.yaml | 4 +- docs/integrations/sources/zendesk-chat.md | 1 + 10 files changed, 211 insertions(+), 38 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/Dockerfile create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/build_customization.py diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index b18881e6f33a..f087303eb3da 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -395,6 +395,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 1.7.0 | [#30526](https://github.com/airbytehq/airbyte/pull/30526) | Implement pre/post install hooks support. | | 1.6.0 | [#30474](https://github.com/airbytehq/airbyte/pull/30474) | Test connector inside their containers. | | 1.5.1 | [#31227](https://github.com/airbytehq/airbyte/pull/31227) | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'. | | 1.5.0 | [#30456](https://github.com/airbytehq/airbyte/pull/30456) | Start building Python connectors using our base images. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py b/airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py new file mode 100644 index 000000000000..03d6f13f9757 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py @@ -0,0 +1,88 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import importlib +from logging import Logger +from types import ModuleType +from typing import List, Optional + +from connector_ops.utils import Connector +from dagger import Container + +BUILD_CUSTOMIZATION_MODULE_NAME = "build_customization" +BUILD_CUSTOMIZATION_SPEC_NAME = f"{BUILD_CUSTOMIZATION_MODULE_NAME}.py" +DEFAULT_MAIN_FILE_NAME = "main.py" + + +def get_build_customization_module(connector: Connector) -> Optional[ModuleType]: + """Import the build_customization.py file from the connector directory if it exists. + Returns: + Optional[ModuleType]: The build_customization.py module if it exists, None otherwise. + """ + build_customization_spec_path = connector.code_directory / BUILD_CUSTOMIZATION_SPEC_NAME + if not build_customization_spec_path.exists(): + return None + build_customization_spec = importlib.util.spec_from_file_location( + f"{connector.code_directory.name}_{BUILD_CUSTOMIZATION_MODULE_NAME}", build_customization_spec_path + ) + build_customization_module = importlib.util.module_from_spec(build_customization_spec) + build_customization_spec.loader.exec_module(build_customization_module) + return build_customization_module + + +def get_main_file_name(connector: Connector) -> str: + """Get the main file name from the build_customization.py module if it exists, DEFAULT_MAIN_FILE_NAME otherwise. + + Args: + connector (Connector): The connector to build. + + Returns: + str: The main file name. + """ + build_customization_module = get_build_customization_module(connector) + if hasattr(build_customization_module, "MAIN_FILE_NAME"): + return build_customization_module.MAIN_FILE_NAME + return DEFAULT_MAIN_FILE_NAME + + +def get_entrypoint(connector: Connector) -> List[str]: + main_file_name = get_main_file_name(connector) + return ["python", f"/airbyte/integration_code/{main_file_name}"] + + +async def pre_install_hooks(connector: Connector, base_container: Container, logger: Logger) -> Container: + """Run the pre_connector_install hook if it exists in the build_customization.py module. + It will mutate the base_container and return it. + + Args: + connector (Connector): The connector to build. + base_container (Container): The base container to mutate. + logger (Logger): The logger to use. + + Returns: + Container: The mutated base_container. + """ + build_customization_module = get_build_customization_module(connector) + if hasattr(build_customization_module, "pre_connector_install"): + base_container = await build_customization_module.pre_connector_install(base_container) + logger.info(f"Connector {connector.technical_name} pre install hook executed.") + return base_container + + +async def post_install_hooks(connector: Connector, connector_container: Container, logger: Logger) -> Container: + """Run the post_connector_install hook if it exists in the build_customization.py module. + It will mutate the connector_container and return it. + + Args: + connector (Connector): The connector to build. + connector_container (Container): The connector container to mutate. + logger (Logger): The logger to use. + + Returns: + Container: The mutated connector_container. + """ + build_customization_module = get_build_customization_module(connector) + if hasattr(build_customization_module, "post_connector_install"): + connector_container = await build_customization_module.post_connector_install(connector_container) + logger.info(f"Connector {connector.technical_name} post install hook executed.") + return connector_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py index c12bbb91f5f1..7c97ca3981f6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py @@ -6,6 +6,7 @@ from dagger import Container, Platform from pipelines.actions.environments import apply_python_development_overrides, with_python_connector_installed from pipelines.bases import StepResult +from pipelines.builds import build_customization from pipelines.builds.common import BuildConnectorImagesBase from pipelines.contexts import ConnectorContext @@ -16,7 +17,6 @@ class BuildConnectorImages(BuildConnectorImagesBase): A spec command is run on the container to validate it was built successfully. """ - DEFAULT_ENTRYPOINT = ["python", "/airbyte/integration_code/main.py"] PATH_TO_INTEGRATION_CODE = "/airbyte/integration_code" async def _build_connector(self, platform: Platform): @@ -35,8 +35,6 @@ def _get_base_container(self, platform: Platform) -> Container: async def _create_builder_container(self, base_container: Container) -> Container: """Pre install the connector dependencies in a builder container. - If a python connectors depends on another local python connector, we need to mount its source in the container - This occurs for the source-file-secure connector for example, which depends on source-file Args: base_container (Container): The base container to use to build the connector. @@ -62,7 +60,11 @@ async def _build_from_base_image(self, platform: Platform) -> Container: """ self.logger.info("Building connector from base image in metadata") base = self._get_base_container(platform) - builder = await self._create_builder_container(base) + customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger) + entrypoint = build_customization.get_entrypoint(self.context.connector) + main_file_name = build_customization.get_main_file_name(self.context.connector) + + builder = await self._create_builder_container(customized_base) # The snake case name of the connector corresponds to the python package name of the connector # We want to mount it to the container under PATH_TO_INTEGRATION_CODE/connector_snake_case_name @@ -70,19 +72,20 @@ async def _build_from_base_image(self, platform: Platform) -> Container: connector_container = ( # copy python dependencies from builder to connector container - base.with_directory("/usr/local", builder.directory("/usr/local")) + customized_base.with_directory("/usr/local", builder.directory("/usr/local")) .with_workdir(self.PATH_TO_INTEGRATION_CODE) - .with_file("main.py", (await self.context.get_connector_dir(include="main.py")).file("main.py")) + .with_file(main_file_name, (await self.context.get_connector_dir(include=main_file_name)).file(main_file_name)) .with_directory( connector_snake_case_name, (await self.context.get_connector_dir(include=connector_snake_case_name)).directory(connector_snake_case_name), ) - .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(self.DEFAULT_ENTRYPOINT)) - .with_entrypoint(self.DEFAULT_ENTRYPOINT) + .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) + .with_entrypoint(entrypoint) .with_label("io.airbyte.version", self.context.connector.metadata["dockerImageTag"]) .with_label("io.airbyte.name", self.context.connector.metadata["dockerRepository"]) ) - return connector_container + customized_connector = await build_customization.post_install_hooks(self.context.connector, connector_container, self.logger) + return customized_connector async def _build_from_dockerfile(self, platform: Platform) -> Container: """Build the connector container using its Dockerfile. diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 1ae1e3f5e963..4967da65b218 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.6.0" +version = "1.7.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py b/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py new file mode 100644 index 000000000000..70840b06ce0b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + """This function will run before the connector installation. + It can mutate the base image container. + + Args: + base_image_container (Container): The base image container to mutate. + + Returns: + Container: The mutated base image container. + """ + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + + +async def post_connector_install(connector_container: Container) -> Container: + """This function will run after the connector installation during the build process. + It can mutate the connector container. + + Args: + connector_container (Container): The connector container to mutate. + + Returns: + Container: The mutated connector container. + """ + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") diff --git a/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py index 576726b54011..d8b042491fb3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py @@ -2,9 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from pathlib import Path + import pytest from pipelines.bases import StepStatus -from pipelines.builds import python_connectors +from pipelines.builds import build_customization, python_connectors from pipelines.contexts import ConnectorContext pytestmark = [ @@ -28,17 +30,41 @@ def test_context_with_connector_without_base_image(self, test_context): return test_context @pytest.fixture - def connector_with_base_image(self, all_connectors): + def connector_with_base_image_no_build_customization(self, all_connectors): for connector in all_connectors: if connector.metadata and connector.metadata.get("connectorBuildOptions", {}).get("baseImage"): - return connector + if not (connector.code_directory / "build_customization.py").exists(): + return connector pytest.skip("No connector with a connectorBuildOptions.baseImage metadata found") @pytest.fixture - def test_context_with_real_connector_using_base_image(self, connector_with_base_image, dagger_client): + def connector_with_base_image_with_build_customization(self, connector_with_base_image_no_build_customization): + dummy_build_customization = (Path(__file__).parent / "dummy_build_customization.py").read_text() + (connector_with_base_image_no_build_customization.code_directory / "build_customization.py").write_text(dummy_build_customization) + yield connector_with_base_image_no_build_customization + (connector_with_base_image_no_build_customization.code_directory / "build_customization.py").unlink() + + @pytest.fixture + def test_context_with_real_connector_using_base_image(self, connector_with_base_image_no_build_customization, dagger_client): + context = ConnectorContext( + pipeline_name="test build", + connector=connector_with_base_image_no_build_customization, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + ) + context.dagger_client = dagger_client + return context + + @pytest.fixture + def test_context_with_real_connector_using_base_image_with_build_customization( + self, connector_with_base_image_with_build_customization, dagger_client + ): context = ConnectorContext( pipeline_name="test build", - connector=connector_with_base_image, + connector=connector_with_base_image_with_build_customization, git_branch="test", git_revision="test", report_output_prefix="test", @@ -87,9 +113,11 @@ async def test_building_from_base_image_for_real(self, test_context_with_real_co step_result = await step._run() step_result.status is StepStatus.SUCCESS built_container = step_result.output_artifact[current_platform] - assert await built_container.env_variable("AIRBYTE_ENTRYPOINT") == " ".join(step.DEFAULT_ENTRYPOINT) + assert await built_container.env_variable("AIRBYTE_ENTRYPOINT") == " ".join( + build_customization.get_entrypoint(step.context.connector) + ) assert await built_container.workdir() == step.PATH_TO_INTEGRATION_CODE - assert await built_container.entrypoint() == step.DEFAULT_ENTRYPOINT + assert await built_container.entrypoint() == build_customization.get_entrypoint(step.context.connector) assert ( await built_container.label("io.airbyte.version") == test_context_with_real_connector_using_base_image.connector.metadata["dockerImageTag"] @@ -99,6 +127,18 @@ async def test_building_from_base_image_for_real(self, test_context_with_real_co == test_context_with_real_connector_using_base_image.connector.metadata["dockerRepository"] ) + async def test_building_from_base_image_with_customization_for_real( + self, test_context_with_real_connector_using_base_image_with_build_customization, current_platform + ): + step = python_connectors.BuildConnectorImages( + test_context_with_real_connector_using_base_image_with_build_customization, current_platform + ) + step_result = await step._run() + step_result.status is StepStatus.SUCCESS + built_container = step_result.output_artifact[current_platform] + assert await built_container.env_variable("MY_PRE_BUILD_ENV_VAR") == "my_pre_build_env_var_value" + assert await built_container.env_variable("MY_POST_BUILD_ENV_VAR") == "my_post_build_env_var_value" + async def test__run_using_base_dockerfile_with_mocks(self, mocker, test_context_with_connector_without_base_image, current_platform): container_built_from_dockerfile = mocker.AsyncMock() mocker.patch.object( diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile deleted file mode 100644 index c7764ec2d2e5..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_zendesk_chat" -ENV AIRBYTE_IMPL_MODULE="source_zendesk_chat" -ENV AIRBYTE_IMPL_PATH="SourceZendeskChat" -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main_dev.py" - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH -COPY main_dev.py ./ -COPY setup.py ./ -RUN pip install . - -ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] - -LABEL io.airbyte.version=0.1.14 -LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py new file mode 100644 index 000000000000..b974ca15ee59 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dagger import Container + +MAIN_FILE_NAME = "main_dev.py" + + +async def pre_connector_install(base_image_container: Container) -> Container: + """This function will run before the connector installation. + We set these environment variable to match what was originally in the Dockerfile. + Disclaimer: I have no idea if these env vars are actually needed. + """ + return await ( + base_image_container.with_env_variable("AIRBYTE_IMPL_MODULE", "source_zendesk_chat").with_env_variable( + "AIRBYTE_IMPL_PATH", "SourceZendeskChat" + ) + ) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 5d9c006e0853..18185bf76a27 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -2,10 +2,12 @@ data: allowedHosts: hosts: - zopim.com + connectorBuildOptions: + baseImage: airbyte/python-connector-base:1.0.0 connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 0.1.14 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-zendesk-chat githubIssueLabel: source-zendesk-chat icon: zendesk-chat.svg diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index eab821953fd8..31cfcba59e2c 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,6 +80,7 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | +| 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | | 0.1.14 | 2023-02-10 | [24190](https://github.com/airbytehq/airbyte/pull/24190) | Fix remove too high min/max from account stream | | 0.1.13 | 2023-02-10 | [22819](https://github.com/airbytehq/airbyte/pull/22819) | Specified date formatting in specification | | 0.1.12 | 2023-01-27 | [22026](https://github.com/airbytehq/airbyte/pull/22026) | Set `AvailabilityStrategy` for streams explicitly to `None` |