airbyte-ci: set user to airbyte on connector build

airbyte-ci/connectors/pipelines/README.md

@@ -850,6 +850,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.43.0  | [#36545](https://github.com/airbytehq/airbyte/pull/36545)  | Switch to `airbyte` user when available in Python base image.                                                                |
 | 4.42.2  | [#48404](https://github.com/airbytehq/airbyte/pull/48404)  | Include `advanced_auth` in spec migration for manifest-only pipeline                                                         |
 | 4.42.1  | [#47316](https://github.com/airbytehq/airbyte/pull/47316)  | Connector testing: skip incremental acceptance test when the connector is not released.                                      |
 | 4.42.0  | [#47386](https://github.com/airbytehq/airbyte/pull/47386)  | Version increment check: make sure consecutive RC remain on the same version.                                                |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py

@@ -5,11 +5,13 @@
 
 from typing import Any
 
+from base_images.bases import AirbyteConnectorBaseImage  # type: ignore
 from dagger import Container, Platform
 from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
 from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.dagger.actions.python.common import apply_python_development_overrides, with_python_connector_installed
+from pipelines.helpers.utils import sh_dash_c
 from pipelines.models.steps import StepResult
 
 
@@ -21,6 +23,21 @@ class BuildConnectorImages(BuildConnectorImagesBase):
 
     context: ConnectorContext
     PATH_TO_INTEGRATION_CODE = "/airbyte/integration_code"
+    USER = AirbyteConnectorBaseImage.USER
+
+    async def _get_image_user(self, base_container: Container) -> str:
+        """If the base image in use has a user named 'airbyte', we will use it as the user for the connector image.
+
+        Args:
+            base_container (Container): The base container to use to build the connector.
+
+        Returns:
+            str: The user to use for the connector image.
+        """
+        users = (await base_container.with_exec(sh_dash_c(["cut -d: -f1 /etc/passwd | sort | uniq"])).stdout()).splitlines()
+        if self.USER in users:
+            return self.USER
+        return "root"
 
     async def _build_connector(self, platform: Platform, *args: Any) -> Container:
         if (
@@ -60,6 +77,7 @@ async def _build_from_base_image(self, platform: Platform) -> Container:
         """
         self.logger.info(f"Building connector from base image in metadata for {platform}")
         base = self._get_base_container(platform)
+        user = await self._get_image_user(base)
         customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger)
         main_file_name = build_customization.get_main_file_name(self.context.connector)
 
@@ -73,16 +91,18 @@ async def _build_from_base_image(self, platform: Platform) -> Container:
             # copy python dependencies from builder to connector container
             customized_base.with_directory("/usr/local", builder.directory("/usr/local"))
             .with_workdir(self.PATH_TO_INTEGRATION_CODE)
-            .with_file(main_file_name, (await self.context.get_connector_dir(include=[main_file_name])).file(main_file_name))
+            .with_exec(["chown", "-R", f"{user}:{user}", "."])
+            .with_file(main_file_name, (await self.context.get_connector_dir(include=[main_file_name])).file(main_file_name), owner=user)
             .with_directory(
                 connector_snake_case_name,
                 (await self.context.get_connector_dir(include=[connector_snake_case_name])).directory(connector_snake_case_name),
+                owner=user,
             )
         )
 
         connector_container = build_customization.apply_airbyte_entrypoint(base_connector_container, self.context.connector)
         customized_connector = await build_customization.post_install_hooks(self.context.connector, connector_container, self.logger)
-        return customized_connector
+        return customized_connector.with_user(user)
 
     async def _build_from_dockerfile(self, platform: Platform) -> Container:
         """Build the connector container using its Dockerfile.

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py

@@ -155,8 +155,8 @@ async def install_testing_environment(
             # Install the connector python package in /test_environment with the extra dependencies
             await pipelines.dagger.actions.python.common.with_python_connector_installed(
                 self.context,
-                # Reset the entrypoint to run non airbyte commands
-                built_connector_container.with_entrypoint([]),
+                # Reset the entrypoint to run non airbyte commands and set the user to root to install the dependencies and access secrets
+                built_connector_container.with_entrypoint([]).with_user("root"),
                 str(self.context.connector.code_directory),
                 additional_dependency_groups=extra_dependencies_names,
             )

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py

@@ -200,6 +200,7 @@ async def with_installed_python_package(
     Returns:
         Container: A python environment container with the python package installed.
     """
+
     container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include)
     local_dependencies = await find_local_python_dependencies(context, package_source_code_path)
 

airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py

@@ -25,11 +25,14 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) ->
 
 
 def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container:
+    user, user_id, group_id = "airbyte", 1000, 1000
+
     integration_base = with_integration_base(context, build_platform)
     yum_packages_to_install = [
         "tar",  # required to untar java connector binary distributions.
         "openssl",  # required because we need to ssh and scp sometimes.
         "findutils",  # required for xargs, which is shipped as part of findutils.
+        "/usr/sbin/adduser",  # required to create a user.
     ]
     return (
         context.dagger_client.container(platform=build_platform)
@@ -51,9 +54,11 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor
                 ]
             )
         )
+        .with_exec(["groupadd", "--gid", str(group_id), "--system", user])
+        .with_exec(["adduser", "--uid", str(user_id), "--gid", str(group_id), "--system", "--no-create-home", user])
         # Add what files we need to the /airbyte directory.
         # Copy base.sh from the airbyte/integration-base image.
-        .with_directory("/airbyte", integration_base.directory("/airbyte"))
+        .with_directory("/airbyte", integration_base.directory("/airbyte"), owner=user)
         .with_workdir("/airbyte")
         # Download a utility jar from the internet.
         .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer"))
@@ -143,7 +148,7 @@ def with_integration_base_java_and_normalization(context: ConnectorContext, buil
 
 async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container:
     application = context.connector.technical_name
-
+    user = "airbyte"
     build_stage = (
         with_integration_base_java(context, build_platform)
         .with_workdir("/airbyte")
@@ -152,6 +157,7 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_
         .with_exec(
             sh_dash_c(
                 [
+                    "chown -R airbyte:airbyte .",
                     f"tar xf {application}.tar --strip-components=1",
                     f"rm -rf {application}.tar",
                 ]
@@ -176,4 +182,8 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_
         .with_exec(sh_dash_c(["mv built_artifacts/* ."]))
         .with_entrypoint(entrypoint)
     )
-    return await finalize_build(context, connector_container)
+    connector_container = await finalize_build(context, connector_container)
+    connector_container = (
+        connector_container.with_exec(["chown", "-R", user, "/airbyte"]).with_exec(["chmod", "777", "/tmp"]).with_user(user)
+    )
+    return connector_container

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.42.2"
+version = "4.43.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-integrations/connectors/source-s3/metadata.yaml

@@ -6,11 +6,11 @@ data:
     hosts:
       - "*.s3.amazonaws.com"
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
+    baseImage: docker.io/airbyte/python-connector-base:3.0.0-rc.1@sha256:ee046486af9ad90b1b248afe5e92846b51375a21463dff1cd377c4f06abb55b5
   connectorSubtype: file
   connectorType: source
   definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
-  dockerImageTag: 4.10.1
+  dockerImageTag: 4.10.2
   dockerRepository: airbyte/source-s3
   documentationUrl: https://docs.airbyte.com/integrations/sources/s3
   githubIssueLabel: source-s3

airbyte-integrations/connectors/source-s3/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.10.1"
+version = "4.10.2"
 name = "source-s3"
 description = "Source implementation for S3."
 authors = [ "Airbyte <contact@airbyte.io>",]