Enable python 3.12

.github/workflows/pythonbuild.yml

@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -39,6 +39,14 @@ jobs:
           make setup
           pip uninstall -y pandas
           pip freeze
+      - name: Run extras unit tests with coverage
+        # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
+        # and https://github.com/pytorch/pytorch/issues/110436
+        if: ${{ matrix.python-version != '3.12' }}
+        env:
+          PYTEST_OPTS: -n2
+        run: |
+          make unit_test_extras_codecov
       - name: Test with coverage
         env:
           PYTEST_OPTS: -n2
@@ -93,7 +101,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -128,10 +136,14 @@ jobs:
         os: [ubuntu-latest]
         # python 3.11 has intermittent issues with the docker build + push step
         # https://github.com/flyteorg/flytekit/actions/runs/5800978835/job/15724237979?pr=1579
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
       - uses: insightsengineering/disk-space-reclaimer@v1
+      # As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
+      # and tags to work.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:

Dockerfile

@@ -1,5 +1,5 @@
 ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-buster
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 MAINTAINER Flyte Team <users@flyte.org>
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

Dockerfile.agent

@@ -1,4 +1,4 @@
-FROM python:3.9-slim-buster
+FROM python:3.9-slim-bookworm
 
 MAINTAINER Flyte Team <users@flyte.org>
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

Dockerfile.dev

@@ -6,7 +6,7 @@
 # $ pyflyte run --image localhost:30000/flytekittest:someversion
 
 ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-buster
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 MAINTAINER Flyte Team <users@flyte.org>
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
@@ -28,10 +28,10 @@ COPY . /flytekit
 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
 RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \
     && pip install --no-cache-dir -e /flytekit \
-    && pip install --no-cache-dir -e /flytekit/plugins/flytekit-k8s-pod \
-    && pip install --no-cache-dir -e /flytekit/plugins/flytekit-deck-standard \
-    && pip install --no-cache-dir -e /flytekit/plugins/flytekit-flyin \
-    && pip install --no-cache-dir scikit-learn \
+        -e /flytekit/plugins/flytekit-k8s-pod \
+        -e /flytekit/plugins/flytekit-deck-standard \
+        -e /flytekit/plugins/flytekit-flyin \
+        scikit-learn \
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/ \

Makefile

@@ -52,12 +52,19 @@ test: lint unit_test
 unit_test_codecov:
 	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test
 
+.PHONY: unit_test_extras_codecov
+unit_test_extras_codecov:
+	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test_extras
+
 .PHONY: unit_test
 unit_test:
-	# Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower)
+	# Skip all extra tests and run them with the necessary env var set so that a working (albeit slower)
 	# library is used to serialize/deserialize protobufs is used.
-	$(PYTEST) -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow --ignore=tests/flytekit/unit/models ${CODECOV_OPTS} && \
-		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python $(PYTEST) tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
+	$(PYTEST) -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models ${CODECOV_OPTS}
+
+.PHONY: unit_test_extras
+unit_test_extras:
+	PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python $(PYTEST) tests/flytekit/unit/extras ${CODECOV_OPTS}
 
 .PHONY: test_serialization_codecov
 test_serialization_codecov:

dev-requirements.in

@@ -19,16 +19,14 @@ IPython
 keyrings.alt
 setuptools_scm
 
-# Only install tensorflow if not running on an arm Mac.
-tensorflow==2.8.1; python_version<'3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
-# Tensorflow release candidate supports python 3.11
-tensorflow==2.13.0; python_version>='3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
-
+# Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003
+tensorflow; python_version<'3.12'
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1; python_version<'3.11'
 # pytorch 2 supports python 3.11
-torch<=2.0.0; python_version>='3.11' or platform_system!='Windows'
+# pytorch 2 does not support 3.12 yet: https://github.com/pytorch/pytorch/issues/110436
+torch; python_version<'3.12'
 
 # TODO: Currently, the python-magic library causes build errors on Windows due to its dependency on DLLs for libmagic.
 # We have temporarily disabled this feature on Windows and are using python-magic for Mac OS and Linux instead.

flytekit/configuration/default_images.py

@@ -8,6 +8,7 @@ class PythonVersion(enum.Enum):
     PYTHON_3_9 = (3, 9)
     PYTHON_3_10 = (3, 10)
     PYTHON_3_11 = (3, 11)
+    PYTHON_3_12 = (3, 12)
 
 
 class DefaultImages(object):
@@ -20,6 +21,7 @@ class DefaultImages(object):
         PythonVersion.PYTHON_3_9: "cr.flyte.org/flyteorg/flytekit:py3.9-",
         PythonVersion.PYTHON_3_10: "cr.flyte.org/flyteorg/flytekit:py3.10-",
         PythonVersion.PYTHON_3_11: "cr.flyte.org/flyteorg/flytekit:py3.11-",
+        PythonVersion.PYTHON_3_12: "cr.flyte.org/flyteorg/flytekit:py3.12-",
     }
 
     @classmethod

plugins/flytekit-deck-standard/setup.py

@@ -8,7 +8,9 @@
     "flytekit",
     "markdown",
     "plotly",
-    "ydata-profiling",
+    # ydata-profiling is not compatible with python 3.12 yet: https://github.com/ydataai/ydata-profiling/issues/1510
+    "ydata-profiling; python_version<'3.12'",
+    "pandas",
     "ipywidgets",
     "pygments",
 ]

plugins/flytekit-flyin/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-buster
+FROM python:3.10-slim-bookworm
 MAINTAINER Flyte Team <users@flyte.org>
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
 WORKDIR /root

plugins/flytekit-sqlalchemy/Dockerfile

@@ -1,5 +1,5 @@
 ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-buster
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 WORKDIR /root
 ENV LANG C.UTF-8

pyproject.toml

@@ -9,7 +9,7 @@ authors = [{ name = "Flyte Contributors", email = "admin@flyte.org" }]
 description = "Flyte SDK for Python"
 license = { text = "Apache-2.0" }
 readme = { file = "README.md", content-type = "text/markdown" }
-requires-python = ">=3.8,<3.12"
+requires-python = ">=3.8,<3.13"
 dependencies = [
     # Please maintain an alphabetical order in the following list
     "adlfs",

tests/flytekit/unit/core/test_type_engine.py

@@ -1407,7 +1407,7 @@ class Bar(DataClassJsonMixin):
 
     pv = Bar(x=3)
     with pytest.raises(
-        TypeTransformerFailedError, match="Type of Val '<class 'int'>' is not an instance of <class 'types.ArgsSchema'>"
+        TypeTransformerFailedError, match="Type of Val '<class 'int'>' is not an instance of <class '.*.ArgsSchema'>"
     ):
         DataclassTransformer().assert_type(gt, pv)
 
@@ -1438,7 +1438,7 @@ class Bar(DataClassJSONMixin):
     pv = Bar(x=3)
     with pytest.raises(
         TypeTransformerFailedError,
-        match="Type of Val '<class 'int'>' is not an instance of <class 'types.ArgsAssert'>",
+        match="Type of Val '<class 'int'>' is not an instance of <class '.*.ArgsAssert'>",
     ):
         DataclassTransformer().assert_type(gt, pv)