ARROW-14217: [Python][CI] Add support for python 3.10

Closes #11316 from kszucs/py310 Lead-authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com> Co-authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
apache · Oct 18, 2021 · 15f4e56 · 15f4e56
1 parent bea1701
commit 15f4e56
Show file tree

Hide file tree

Showing 15 changed files with 67 additions and 33 deletions.
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -88,7 +88,8 @@ ARG python=3.6
 RUN (if "%python%"=="3.6" setx PYTHON_VERSION 3.6.8) & \
     (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.4) & \
     (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.6) & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.1)
+    (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.1) & \
+    (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.0)
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
 RUN python -m pip install -U pip
 

diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh
@@ -28,7 +28,8 @@ declare -A versions
 versions=([3.6]=3.6.8
           [3.7]=3.7.9
           [3.8]=3.8.10
-          [3.9]=3.9.6)
+          [3.9]=3.9.6,
+          [3.10]=3.10.0)
 
 if [ "$#" -ne 2 ]; then
   echo "Usage: $0 <platform> <version>"

diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh
@@ -51,7 +51,7 @@ export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data
 
 if [ "${INSTALL_PYARROW}" == "ON" ]; then
   # Install the built wheels
-  pip install ${source_dir}/python/repaired_wheels/*.whl
+  pip install --force-reinstall ${source_dir}/python/repaired_wheels/*.whl
 fi
 
 if [ "${CHECK_IMPORTS}" == "ON" ]; then

diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
@@ -162,12 +162,41 @@ Status IntegerOverflowStatus(PyObject* obj, const std::string& overflow_message)
   }
 }
 
+Result<OwnedRef> PyObjectToPyInt(PyObject* obj) {
+  // Try to call __index__ or __int__ on `obj`
+  // (starting from Python 3.10, the latter isn't done anymore by PyLong_AsLong*).
+  OwnedRef ref(PyNumber_Index(obj));
+  if (ref) {
+    return ref;
+  }
+  PyErr_Clear();
+  const auto nb = Py_TYPE(obj)->tp_as_number;
+  if (nb && nb->nb_int) {
+    ref.reset(nb->nb_int(obj));
+    if (!ref) {
+      RETURN_IF_PYERROR();
+    }
+    DCHECK(ref);
+    return ref;
+  }
+  return Status::TypeError(
+      "object of type ",
+      PyObject_StdStringRepr(reinterpret_cast<PyObject*>(Py_TYPE(obj))),
+      " cannot be converted to int");
+}
+
 // Extract C signed int from Python object
 template <typename Int, enable_if_t<std::is_signed<Int>::value, Int> = 0>
 Status CIntFromPythonImpl(PyObject* obj, Int* out, const std::string& overflow_message) {
   static_assert(sizeof(Int) <= sizeof(long long),  // NOLINT
                 "integer type larger than long long");
 
+  OwnedRef ref;
+  if (!PyLong_Check(obj)) {
+    ARROW_ASSIGN_OR_RAISE(ref, PyObjectToPyInt(obj));
+    obj = ref.obj();
+  }
+
   if (sizeof(Int) > sizeof(long)) {  // NOLINT
     const auto value = PyLong_AsLongLong(obj);
     if (ARROW_PREDICT_FALSE(value == -1)) {
@@ -199,15 +228,11 @@ Status CIntFromPythonImpl(PyObject* obj, Int* out, const std::string& overflow_m
                 "integer type larger than unsigned long long");
 
   OwnedRef ref;
-  // PyLong_AsUnsignedLong() and PyLong_AsUnsignedLongLong() don't handle
-  // conversion from non-ints (e.g. np.uint64), so do it ourselves
   if (!PyLong_Check(obj)) {
-    ref.reset(PyNumber_Index(obj));
-    if (!ref) {
-      RETURN_IF_PYERROR();
-    }
+    ARROW_ASSIGN_OR_RAISE(ref, PyObjectToPyInt(obj));
     obj = ref.obj();
   }
+
   if (sizeof(Int) > sizeof(unsigned long)) {  // NOLINT
     const auto value = PyLong_AsUnsignedLongLong(obj);
     if (ARROW_PREDICT_FALSE(value == static_cast<decltype(value)>(-1))) {

diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
@@ -206,14 +206,16 @@ class PyValue {
   }
 
   template <typename T>
-  static enable_if_integer<T, Result<typename T::c_type>> Convert(const T*, const O&,
+  static enable_if_integer<T, Result<typename T::c_type>> Convert(const T* type, const O&,
                                                                   I obj) {
     typename T::c_type value;
     auto status = internal::CIntFromPython(obj, &value);
     if (ARROW_PREDICT_TRUE(status.ok())) {
       return value;
     } else if (!internal::PyIntScalar_Check(obj)) {
-      return internal::InvalidValue(obj, "tried to convert to int");
+      std::stringstream ss;
+      ss << "tried to convert to " << type->ToString();
+      return internal::InvalidValue(obj, ss.str());
     } else {
       return status;
     }

diff --git a/dev/tasks/conda-recipes/azure.yml b/dev/tasks/conda-recipes/azure.yml
diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml
@@ -26,7 +26,7 @@ jobs:
   {% if env is defined %}
     env:
     {% for key, value in env.items() %}
-      {{ key }}: {{ value }}
+      {{ key }}: "{{ value }}"
     {% endfor %}
   {% endif %}
     steps:

diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml
@@ -26,7 +26,7 @@ jobs:
     env:
       # archery uses these environment variables
       ARCH: amd64
-      PYTHON: {{ python_version }}
+      PYTHON: "{{ python_version }}"
 
     steps:
       {{ macros.github_checkout_arrow()|indent }}

diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml
@@ -25,7 +25,7 @@ jobs:
     runs-on: windows-2019
     env:
       # archery uses this environment variable
-      PYTHON: {{ python_version }}
+      PYTHON: "{{ python_version }}"
       # this is a private repository at the moment (mostly because of licensing
       # consideration of windows images with visual studio), but anyone can
       # recreate the image by manually building it via:

diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml
@@ -42,7 +42,7 @@ env:
     - TRAVIS_TAG={{ task.tag }}
     # archery uses these environment variables
     - ARCH=arm64v8
-    - PYTHON={{ python_version }}
+    - PYTHON="{{ python_version }}"
 
 before_script:
   - set -e

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -334,7 +334,8 @@ tasks:
 {% for python_version, python_tag, abi_tag in [("3.6", "cp36", "cp36m"),
                                                ("3.7", "cp37", "cp37m"),
                                                ("3.8", "cp38", "cp38"),
-                                               ("3.9", "cp39", "cp39")] %}
+                                               ("3.9", "cp39", "cp39"),
+                                               ("3.10", "cp310", "cp310")] %}
 
 {############################## Wheel Linux ##################################}
 
@@ -347,7 +348,7 @@ tasks:
     ci: {{ ci }}
     template: python-wheels/{{ ci }}.linux.{{ arch }}.yml
     params:
-      python_version: {{ python_version }}
+      python_version: "{{ python_version }}"
       manylinux_version: {{ manylinux }}
     artifacts:
       - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl
@@ -366,7 +367,7 @@ tasks:
     template: python-wheels/github.osx.amd64.yml
     params:
       vcpkg_version: "2021.04.30"
-      python_version: {{ python_version }}
+      python_version: "{{ python_version }}"
       macos_deployment_target: {{ macos_version }}
       arrow_s3: {{ arrow_s3 }}
     artifacts:
@@ -380,7 +381,7 @@ tasks:
     ci: github
     template: python-wheels/github.windows.yml
     params:
-      python_version: {{ python_version }}
+      python_version: "{{ python_version }}"
     artifacts:
       - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl
 
@@ -402,7 +403,7 @@ tasks:
     artifacts:
       - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl
 
-{% for python_version, python_tag in [("3.9", "cp39")] %}
+{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %}
   wheel-macos-big-sur-{{ python_tag }}-arm64:
     ci: github
     template: python-wheels/github.osx.arm64.yml
@@ -940,13 +941,13 @@ tasks:
         UBUNTU: 20.04
       image: ubuntu-cpp-thread-sanitizer
 
-{% for python_version in ["3.6", "3.7", "3.8", "3.9"] %}
+{% for python_version in ["3.6", "3.7", "3.8", "3.9", "3.10"] %}
   test-conda-python-{{ python_version }}:
     ci: github
     template: docker-tests/github.linux.yml
     params:
       env:
-        PYTHON: {{ python_version }}
+        PYTHON: "{{ python_version }}"
       image: conda-python
 {% endfor %}
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -754,7 +754,7 @@ services:
       args:
         arch_alias: ${ARCH_ALIAS}
         arch_short_alias: ${ARCH_SHORT_ALIAS}
-        base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2020-12-03-912b0de
+        base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
       context: .
@@ -777,7 +777,7 @@ services:
       args:
         arch_alias: ${ARCH_ALIAS}
         arch_short_alias: ${ARCH_SHORT_ALIAS}
-        base:  quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2020-11-11-bc8ce45
+        base:  quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
       context: .

diff --git a/python/pyarrow/error.pxi b/python/pyarrow/error.pxi
@@ -175,7 +175,8 @@ def enable_signal_handlers(c_bool enable):
 
 # Whether we need a workaround for https://bugs.python.org/issue42248
 have_signal_refcycle = (sys.version_info < (3, 8, 10) or
-                        (3, 9) <= sys.version_info < (3, 9, 5))
+                        (3, 9) <= sys.version_info < (3, 9, 5) or
+                        sys.version_info[:2] == (3, 10))
 
 cdef class SignalStopHandler:
     cdef:

diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt
@@ -2,10 +2,13 @@ cython>=0.29.11
 setuptools>=58
 setuptools_scm
 wheel
-numpy==1.19.4; platform_system == "Linux"   and platform_machine == "aarch64"
+numpy==1.19.4; platform_system == "Linux"   and platform_machine == "aarch64" and python_version <= "3.9"
+numpy==1.21.2; platform_system == "Linux"   and platform_machine == "aarch64" and python_version >  "3.9"
 numpy==1.16.6; platform_system == "Linux"   and platform_machine != "aarch64" and python_version <  "3.9"
-numpy==1.19.4; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.9"
-numpy==1.21.0; platform_system == "Darwin"  and platform_machine == "arm64"
+numpy==1.19.4; platform_system == "Linux"   and platform_machine != "aarch64" and python_version == "3.9"
+numpy==1.21.2; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >  "3.9"
+numpy==1.21.0; platform_system == "Darwin"  and platform_machine == "arm64"   and python_version <= "3.9"
+numpy==1.21.2; platform_system == "Darwin"  and platform_machine == "arm64"   and python_version >  "3.9"
 numpy==1.16.6; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version <  "3.8"
 numpy==1.19.4; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version >= "3.8"
 numpy==1.16.6; platform_system == "Windows"                                   and python_version <  "3.9"

diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt
@@ -7,14 +7,14 @@ pytest-lazy-fixture
 pytz
 
 numpy==1.19.5; platform_system == "Linux"   and platform_machine == "aarch64" and python_version <  "3.7"
-numpy==1.20.3; platform_system == "Linux"   and platform_machine == "aarch64" and python_version >= "3.7"
+numpy==1.21.2; platform_system == "Linux"   and platform_machine == "aarch64" and python_version >= "3.7"
 numpy==1.19.5; platform_system == "Linux"   and platform_machine != "aarch64" and python_version <  "3.9"
-numpy==1.20.3; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.9"
-numpy==1.21.0; platform_system == "Darwin"  and platform_machine == "arm64"
+numpy==1.21.2; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.9"
+numpy==1.21.2; platform_system == "Darwin"  and platform_machine == "arm64"
 numpy==1.19.5; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version <  "3.9"
-numpy==1.20.3; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version >= "3.9"
+numpy==1.21.2; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version >= "3.9"
 numpy==1.19.5; platform_system == "Windows"                                   and python_version <  "3.9"
-numpy==1.20.3; platform_system == "Windows"                                   and python_version >= "3.9"
+numpy==1.21.2; platform_system == "Windows"                                   and python_version >= "3.9"
 
 pandas<1.1.0;  platform_system == "Linux"   and platform_machine != "aarch64" and python_version <  "3.8"
 pandas;        platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.8"