From 5121ff33c8f7c2a2972f5f1602afae7f88ce157f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 27 Mar 2024 11:10:52 +0100 Subject: [PATCH 1/4] airbyte-ci: set user to airbyte on connector build --- airbyte-ci/connectors/pipelines/README.md | 1 + .../build_image/steps/python_connectors.py | 24 +++++++++++++++++-- .../test/steps/python_connectors.py | 4 ++-- .../pipelines/dagger/actions/python/common.py | 1 + .../pipelines/dagger/containers/java.py | 16 ++++++++++--- .../connectors/pipelines/pyproject.toml | 2 +- .../connectors/source-s3/metadata.yaml | 4 ++-- .../connectors/source-s3/pyproject.toml | 2 +- 8 files changed, 43 insertions(+), 11 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 87af15d61291..920782c1369f 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -850,6 +850,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only | Version | PR | Description | | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.43.0 | [#36545](https://github.com/airbytehq/airbyte/pull/36545) | Switch to `airbyte` user when available in Python base image. | | 4.42.2 | [#48404](https://github.com/airbytehq/airbyte/pull/48404) | Include `advanced_auth` in spec migration for manifest-only pipeline | | 4.42.1 | [#47316](https://github.com/airbytehq/airbyte/pull/47316) | Connector testing: skip incremental acceptance test when the connector is not released. | | 4.42.0 | [#47386](https://github.com/airbytehq/airbyte/pull/47386) | Version increment check: make sure consecutive RC remain on the same version. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py index 3e81983bc86e..f61bfe656697 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py @@ -5,11 +5,13 @@ from typing import Any +from base_images.bases import AirbyteConnectorBaseImage # type: ignore from dagger import Container, Platform from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python.common import apply_python_development_overrides, with_python_connector_installed +from pipelines.helpers.utils import sh_dash_c from pipelines.models.steps import StepResult @@ -21,6 +23,21 @@ class BuildConnectorImages(BuildConnectorImagesBase): context: ConnectorContext PATH_TO_INTEGRATION_CODE = "/airbyte/integration_code" + USER = AirbyteConnectorBaseImage.USER + + async def _get_image_user(self, base_container: Container) -> str: + """If the base image in use has a user named 'airbyte', we will use it as the user for the connector image. + + Args: + base_container (Container): The base container to use to build the connector. + + Returns: + str: The user to use for the connector image. + """ + users = (await base_container.with_exec(sh_dash_c(["cut -d: -f1 /etc/passwd | sort | uniq"])).stdout()).splitlines() + if self.USER in users: + return self.USER + return "root" async def _build_connector(self, platform: Platform, *args: Any) -> Container: if ( @@ -60,6 +77,7 @@ async def _build_from_base_image(self, platform: Platform) -> Container: """ self.logger.info(f"Building connector from base image in metadata for {platform}") base = self._get_base_container(platform) + user = await self._get_image_user(base) customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger) main_file_name = build_customization.get_main_file_name(self.context.connector) @@ -73,16 +91,18 @@ async def _build_from_base_image(self, platform: Platform) -> Container: # copy python dependencies from builder to connector container customized_base.with_directory("/usr/local", builder.directory("/usr/local")) .with_workdir(self.PATH_TO_INTEGRATION_CODE) - .with_file(main_file_name, (await self.context.get_connector_dir(include=[main_file_name])).file(main_file_name)) + .with_exec(["chown", "-R", f"{user}:{user}", "."]) + .with_file(main_file_name, (await self.context.get_connector_dir(include=[main_file_name])).file(main_file_name), owner=user) .with_directory( connector_snake_case_name, (await self.context.get_connector_dir(include=[connector_snake_case_name])).directory(connector_snake_case_name), + owner=user, ) ) connector_container = build_customization.apply_airbyte_entrypoint(base_connector_container, self.context.connector) customized_connector = await build_customization.post_install_hooks(self.context.connector, connector_container, self.logger) - return customized_connector + return customized_connector.with_user(user) async def _build_from_dockerfile(self, platform: Platform) -> Container: """Build the connector container using its Dockerfile. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 817a662cd703..dea503120da3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -155,8 +155,8 @@ async def install_testing_environment( # Install the connector python package in /test_environment with the extra dependencies await pipelines.dagger.actions.python.common.with_python_connector_installed( self.context, - # Reset the entrypoint to run non airbyte commands - built_connector_container.with_entrypoint([]), + # Reset the entrypoint to run non airbyte commands and set the user to root to install the dependencies and access secrets + built_connector_container.with_entrypoint([]).with_user("root"), str(self.context.connector.code_directory), additional_dependency_groups=extra_dependencies_names, ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index e0d3990d558b..04eb0eeef9d8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -200,6 +200,7 @@ async def with_installed_python_package( Returns: Container: A python environment container with the python package installed. """ + container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include) local_dependencies = await find_local_python_dependencies(context, package_source_code_path) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index ad7fb7e4bcdf..8239ac9b9006 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -25,11 +25,14 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) -> def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container: + user, user_id, group_id = "airbyte", 1000, 1000 + integration_base = with_integration_base(context, build_platform) yum_packages_to_install = [ "tar", # required to untar java connector binary distributions. "openssl", # required because we need to ssh and scp sometimes. "findutils", # required for xargs, which is shipped as part of findutils. + "/usr/sbin/adduser", # required to create a user. ] return ( context.dagger_client.container(platform=build_platform) @@ -51,9 +54,11 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor ] ) ) + .with_exec(["groupadd", "--gid", str(group_id), "--system", user]) + .with_exec(["adduser", "--uid", str(user_id), "--gid", str(group_id), "--system", "--no-create-home", user]) # Add what files we need to the /airbyte directory. # Copy base.sh from the airbyte/integration-base image. - .with_directory("/airbyte", integration_base.directory("/airbyte")) + .with_directory("/airbyte", integration_base.directory("/airbyte"), owner=user) .with_workdir("/airbyte") # Download a utility jar from the internet. .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer")) @@ -143,7 +148,7 @@ def with_integration_base_java_and_normalization(context: ConnectorContext, buil async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container: application = context.connector.technical_name - + user = "airbyte" build_stage = ( with_integration_base_java(context, build_platform) .with_workdir("/airbyte") @@ -152,6 +157,7 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ .with_exec( sh_dash_c( [ + "chown -R airbyte:airbyte .", f"tar xf {application}.tar --strip-components=1", f"rm -rf {application}.tar", ] @@ -176,4 +182,8 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ .with_exec(sh_dash_c(["mv built_artifacts/* ."])) .with_entrypoint(entrypoint) ) - return await finalize_build(context, connector_container) + connector_container = await finalize_build(context, connector_container) + connector_container = ( + connector_container.with_exec(["chown", "-R", user, "/airbyte"]).with_exec(["chmod", "777", "/tmp"]).with_user(user) + ) + return connector_container diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 53a1c2fac474..3ab8d5d5c1de 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.42.2" +version = "4.43.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index d5c5e90bb720..620175d22494 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -6,11 +6,11 @@ data: hosts: - "*.s3.amazonaws.com" connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916 + baseImage: docker.io/airbyte/python-connector-base:3.0.0-rc.1@sha256:ee046486af9ad90b1b248afe5e92846b51375a21463dff1cd377c4f06abb55b5 connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.10.1 + dockerImageTag: 4.10.2 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 diff --git a/airbyte-integrations/connectors/source-s3/pyproject.toml b/airbyte-integrations/connectors/source-s3/pyproject.toml index e038af3f9f9d..5e8804d5db8e 100644 --- a/airbyte-integrations/connectors/source-s3/pyproject.toml +++ b/airbyte-integrations/connectors/source-s3/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.10.1" +version = "4.10.2" name = "source-s3" description = "Source implementation for S3." authors = [ "Airbyte ",] From 2f8d40e417b42acf3350cab0df45f1d995cafa62 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 21 Nov 2024 20:17:18 +0100 Subject: [PATCH 2/4] test on source-mailchimp --- .../connectors/source-mailchimp/metadata.yaml | 4 ++-- .../connectors/source-mailchimp/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 2d3ef01d95dc..9458e64b64ff 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -7,11 +7,11 @@ data: - "*.api.mailchimp.com" - "login.mailchimp.com" connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916 + baseImage: docker.io/airbyte/python-connector-base:3.0.0-rc.1@sha256:ee046486af9ad90b1b248afe5e92846b51375a21463dff1cd377c4f06abb55b5 connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 2.0.19 + dockerImageTag: 2.0.20 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/pyproject.toml b/airbyte-integrations/connectors/source-mailchimp/pyproject.toml index f2c929e7e9fe..1781252fa81c 100644 --- a/airbyte-integrations/connectors/source-mailchimp/pyproject.toml +++ b/airbyte-integrations/connectors/source-mailchimp/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.0.19" +version = "2.0.20" name = "source-mailchimp" description = "Source implementation for Mailchimp." authors = [ "Airbyte ",] From 7fcd4c4498cd3cf7c85c8853718bb5a1bec3713b Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 22 Nov 2024 08:34:55 +0100 Subject: [PATCH 3/4] revert connector changes --- .../connectors/source-mailchimp/metadata.yaml | 4 ++-- .../connectors/source-mailchimp/pyproject.toml | 2 +- airbyte-integrations/connectors/source-s3/metadata.yaml | 4 ++-- airbyte-integrations/connectors/source-s3/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 9458e64b64ff..2d3ef01d95dc 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -7,11 +7,11 @@ data: - "*.api.mailchimp.com" - "login.mailchimp.com" connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:3.0.0-rc.1@sha256:ee046486af9ad90b1b248afe5e92846b51375a21463dff1cd377c4f06abb55b5 + baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916 connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 2.0.20 + dockerImageTag: 2.0.19 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/pyproject.toml b/airbyte-integrations/connectors/source-mailchimp/pyproject.toml index 1781252fa81c..f2c929e7e9fe 100644 --- a/airbyte-integrations/connectors/source-mailchimp/pyproject.toml +++ b/airbyte-integrations/connectors/source-mailchimp/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.0.20" +version = "2.0.19" name = "source-mailchimp" description = "Source implementation for Mailchimp." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 620175d22494..d5c5e90bb720 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -6,11 +6,11 @@ data: hosts: - "*.s3.amazonaws.com" connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:3.0.0-rc.1@sha256:ee046486af9ad90b1b248afe5e92846b51375a21463dff1cd377c4f06abb55b5 + baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916 connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.10.2 + dockerImageTag: 4.10.1 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 diff --git a/airbyte-integrations/connectors/source-s3/pyproject.toml b/airbyte-integrations/connectors/source-s3/pyproject.toml index 5e8804d5db8e..e038af3f9f9d 100644 --- a/airbyte-integrations/connectors/source-s3/pyproject.toml +++ b/airbyte-integrations/connectors/source-s3/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.10.2" +version = "4.10.1" name = "source-s3" description = "Source implementation for S3." authors = [ "Airbyte ",] From de8d7ef03744c633f0c8c0c754f0dbfacd7c5512 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 22 Nov 2024 08:44:41 +0100 Subject: [PATCH 4/4] revert java changes --- .../pipelines/dagger/containers/java.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index 8239ac9b9006..ad7fb7e4bcdf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -25,14 +25,11 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) -> def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container: - user, user_id, group_id = "airbyte", 1000, 1000 - integration_base = with_integration_base(context, build_platform) yum_packages_to_install = [ "tar", # required to untar java connector binary distributions. "openssl", # required because we need to ssh and scp sometimes. "findutils", # required for xargs, which is shipped as part of findutils. - "/usr/sbin/adduser", # required to create a user. ] return ( context.dagger_client.container(platform=build_platform) @@ -54,11 +51,9 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor ] ) ) - .with_exec(["groupadd", "--gid", str(group_id), "--system", user]) - .with_exec(["adduser", "--uid", str(user_id), "--gid", str(group_id), "--system", "--no-create-home", user]) # Add what files we need to the /airbyte directory. # Copy base.sh from the airbyte/integration-base image. - .with_directory("/airbyte", integration_base.directory("/airbyte"), owner=user) + .with_directory("/airbyte", integration_base.directory("/airbyte")) .with_workdir("/airbyte") # Download a utility jar from the internet. .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer")) @@ -148,7 +143,7 @@ def with_integration_base_java_and_normalization(context: ConnectorContext, buil async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container: application = context.connector.technical_name - user = "airbyte" + build_stage = ( with_integration_base_java(context, build_platform) .with_workdir("/airbyte") @@ -157,7 +152,6 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ .with_exec( sh_dash_c( [ - "chown -R airbyte:airbyte .", f"tar xf {application}.tar --strip-components=1", f"rm -rf {application}.tar", ] @@ -182,8 +176,4 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ .with_exec(sh_dash_c(["mv built_artifacts/* ."])) .with_entrypoint(entrypoint) ) - connector_container = await finalize_build(context, connector_container) - connector_container = ( - connector_container.with_exec(["chown", "-R", user, "/airbyte"]).with_exec(["chmod", "777", "/tmp"]).with_user(user) - ) - return connector_container + return await finalize_build(context, connector_container)