airbyte-ci: tweaks for cross-repo compatibility (#41541)

Co-authored-by: alafanechere <augustin.lafanechere@gmail.com>

airbyte-ci/connectors/connector_ops/README.md

@@ -37,5 +37,7 @@ poetry run pytest
 ```
 
 ## Changelog
+- 0.6.1: Simplified gradle dependency discovery logic.
+- 0.6.0: Added manifest-only build.
 - 0.5.0: Added `cloud_usage` property to `Connector` class.
 - 0.4.0: Removed acceptance test configuration and allowed hosts checks as they're not used.

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -207,26 +207,6 @@ def parse_gradle_dependencies(build_file: Path) -> Tuple[List[Path], List[Path]]
     return project_dependencies, test_dependencies
 
 
-def get_local_cdk_gradle_dependencies(with_test_dependencies: bool) -> List[Path]:
-    """Recursively retrieve all transitive dependencies of a Gradle project.
-
-    Args:
-        with_test_dependencies: True to include test dependencies.
-
-    Returns:
-        List[Path]: All dependencies of the project.
-    """
-    base_path = Path("airbyte-cdk/java/airbyte-cdk")
-    found: List[Path] = [base_path]
-    for submodule in ["core", "db-sources", "db-destinations"]:
-        found.append(base_path / submodule)
-        project_dependencies, test_dependencies = parse_gradle_dependencies(base_path / Path(submodule) / Path("build.gradle"))
-        found += project_dependencies
-        if with_test_dependencies:
-            found += test_dependencies
-    return list(set(found))
-
-
 def get_all_gradle_dependencies(
     build_file: Path, with_test_dependencies: bool = True, found_dependencies: Optional[List[Path]] = None
 ) -> List[Path]:
@@ -242,12 +222,6 @@ def get_all_gradle_dependencies(
     if found_dependencies is None:
         found_dependencies = []
     project_dependencies, test_dependencies = parse_gradle_dependencies(build_file)
-
-    # Since first party project folders are transitive (compileOnly) in the
-    # CDK, we always need to add them as the project dependencies.
-    project_dependencies += get_local_cdk_gradle_dependencies(False)
-    test_dependencies += get_local_cdk_gradle_dependencies(with_test_dependencies=True)
-
     all_dependencies = project_dependencies + test_dependencies if with_test_dependencies else project_dependencies
     for dependency_path in all_dependencies:
         if dependency_path not in found_dependencies and Path(dependency_path / "build.gradle").exists():
@@ -685,6 +659,7 @@ def __repr__(self) -> str:
     def get_local_dependency_paths(self, with_test_dependencies: bool = True) -> Set[Path]:
         dependencies_paths = []
         if self.language == ConnectorLanguage.JAVA:
+            dependencies_paths += [Path("./airbyte-cdk/java/airbyte-cdk")]
             dependencies_paths += get_all_gradle_dependencies(
                 self.code_directory / "build.gradle", with_test_dependencies=with_test_dependencies
             )

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector_ops"
-version = "0.6.0"
+version = "0.6.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/README.md

@@ -767,6 +767,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.23.1  | [#41541](https://github.com/airbytehq/airbyte/pull/41541)  | Add support for submodule use-case.                                                                                          |
 | 4.23.0  | [#39906](https://github.com/airbytehq/airbyte/pull/39906)  | Add manifest only build pipeline                                                          |
 | 4.22.0  | [#41623](https://github.com/airbytehq/airbyte/pull/41623)  | Make `airbyte-ci` run on private forks.                                                                                      |
 | 4.21.1  | [#41029](https://github.com/airbytehq/airbyte/pull/41029)  | `up-to-date`: mount local docker config to `Syft` to pull private images and benefit from increased DockerHub rate limits.   |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py

@@ -10,7 +10,7 @@
 import requests
 from dagger import CacheSharingMode, CacheVolume, Container, ExecError
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.consts import AMAZONCORRETTO_IMAGE
+from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME, AMAZONCORRETTO_IMAGE
 from pipelines.dagger.actions import secrets
 from pipelines.hacks import never_fail_exec
 from pipelines.helpers.utils import dagger_directory_as_zip_file, sh_dash_c
@@ -96,6 +96,8 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
             "tools/lib/lib.sh",
             "pyproject.toml",
         ] + self.build_include
+        # Support the edge case where the airbyte repo is used as a git submodule.
+        include += [f"{AIRBYTE_SUBMODULE_DIR_NAME}/{i}" for i in include]
 
         yum_packages_to_install = [
             "docker",  # required by :integrationTestJava.

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -65,6 +65,7 @@
 DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://upload.pypi.org/legacy/"
 DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL = "https://pypi.org/pypi"
 MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS = "airbyte-connector-testing-secret-store"
+AIRBYTE_SUBMODULE_DIR_NAME = "airbyte-submodule"
 
 
 class CIContext(str, Enum):

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py

@@ -9,6 +9,7 @@
 import toml
 from dagger import Container, Directory
 from pipelines.airbyte_ci.connectors.context import PipelineContext
+from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME
 from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package
 from pipelines.dagger.actions.system.common import with_debian_packages
 from pipelines.dagger.containers.python import with_python_base
@@ -40,6 +41,8 @@ async def find_local_dependencies_in_pyproject_toml(
     for value in pyproject_content["tool"]["poetry"]["dependencies"].values():
         if isinstance(value, dict) and "path" in value:
             local_dependency_path = str((Path(pyproject_file_path) / Path(value["path"])).resolve().relative_to(Path.cwd()))
+            # Support the edge case where the airbyte repo is used as a git submodule.
+            local_dependency_path = local_dependency_path.removeprefix(f"{AIRBYTE_SUBMODULE_DIR_NAME}/")
             local_dependency_paths.append(local_dependency_path)
 
             # Ensure we parse the child dependencies

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.23.0"
+version = "4.23.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-integrations/connectors/build.gradle

@@ -60,9 +60,6 @@ tasks.named('clean').configure {
 }
 
 allprojects {
-    // Evaluate CDK project before evaluating the connector.
-    evaluationDependsOn(':airbyte-cdk:java:airbyte-cdk')
-
     // Adds airbyte-ci task.
     def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs ->
         def task = tasks.register(taskName, Exec) {

settings.gradle

@@ -146,12 +146,14 @@ rootProject.name = 'airbyte'
 
 // Include all java CDK modules.
 def cdkPath = rootDir.toPath().resolve('airbyte-cdk/java/airbyte-cdk')
-cdkPath.eachDir { dir ->
-    def buildFiles = file(dir).list { file, name -> name == "build.gradle" }
-    if (buildFiles.length == 1) {
-        def path = ":airbyte-cdk:java:airbyte-cdk:${dir.getFileName()}"
-        include path
-        project(path).name = "airbyte-cdk-${dir.getFileName()}"
+if (cdkPath.toFile().exists()) {
+    cdkPath.eachDir { dir ->
+        def buildFiles = file(dir).list { file, name -> name == "build.gradle" }
+        if (buildFiles.length == 1) {
+            def path = ":airbyte-cdk:java:airbyte-cdk:${dir.getFileName()}"
+            include path
+            project(path).name = "airbyte-cdk-${dir.getFileName()}"
+        }
     }
 }