gradle: search for python3.11 binary for AL2023 support (#31227)

airbytehq · Oct 10, 2023 · ae2ca40 · ae2ca40
1 parent 898846d
commit ae2ca40
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 3 deletions.
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -395,6 +395,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 1.5.1   | [#31227](https://github.com/airbytehq/airbyte/pull/31227)  | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'.     |
 | 1.5.0   | [#30456](https://github.com/airbytehq/airbyte/pull/30456)  | Start building Python connectors using our base images.                                                   |
 | 1.4.6   | [ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date                                                            |
 | 1.4.5   | [#31133](https://github.com/airbytehq/airbyte/pull/31133)  | Fix bug when building containers using `with_integration_base_java_and_normalization`.                    |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/gradle.py
@@ -90,7 +90,7 @@ async def _run(self) -> StepResult:
             "findutils",  # gradle requires xargs, which is shipped in findutils.
             "jq",  # required by :airbyte-connector-test-harnesses:acceptance-test-harness to inspect docker images.
             "npm",  # required by :format.
-            "pip",  # required by :format.
+            "python3.11-pip",  # required by :format.
             "rsync",  # required for gradle cache synchronization.
         ]
 

diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py
@@ -55,7 +55,7 @@ class UnitTests(GradleTask):
     """A step to run unit tests for Java connectors."""
 
     title = "Java Connector Unit Tests"
-    gradle_task_name = "check"
+    gradle_task_name = "test"
     bind_to_docker_host = True
 
 

diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "1.5.0"
+version = "1.5.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

diff --git a/build.gradle b/build.gradle
@@ -159,6 +159,18 @@ python {
     envPath = '.venv'
     minPythonVersion = '3.10' // should be 3.10 for local development
 
+    // Amazon Linux support.
+    // The airbyte-ci tool runs gradle tasks in AL2023-based containers.
+    // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly.
+    // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details.
+    try {
+        if ("python3.11 --version".execute().waitFor() == 0) {
+            // python3.11 definitely exists at this point, use it instead of 'python3'.
+            pythonBinary "python3.11"
+        }
+    } catch (IOException _) {
+        // Swallow exception if python3.11 is not installed.
+    }
     // Pyenv support.
     try {
         def pyenvRoot = "pyenv root".execute()

diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle
@@ -80,6 +80,18 @@ class AirbytePythonPlugin implements Plugin<Project> {
             envPath = venvDirectoryName
             minPythonVersion '3.10'
 
+            // Amazon Linux support.
+            // The airbyte-ci tool runs gradle tasks in AL2023-based containers.
+            // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly.
+            // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details.
+            try {
+                if ("python3.11 --version".execute().waitFor() == 0) {
+                    // python3.11 definitely exists at this point, use it instead of 'python3'.
+                    pythonBinary "python3.11"
+                }
+            } catch (IOException _) {
+                // Swallow exception if python3.11 is not installed.
+            }
             // Pyenv support.
             try {
                 def pyenvRoot = "pyenv root".execute()