Update pandas minimum requirement for Python 3.12 (#40272)

Pandas minimum requirement for Python 3.12 should be changed to be higher than >=2.1,1 because 2.1.1 version does not have proper exclusions for numpy<2.0.0 and on Python 3.12 it will install incompatible numpy 2.0.0 with Pandas 2.1.1 which fails with ``` numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject ``` This is tracked in numpy/numpy#26710 but until (maybe) it is solved in numpy/pandas, we should limit it in Airflow.
apache · Jun 17, 2024 · 81c331e · 81c331e
1 parent 841b28c
commit 81c331e
Show file tree

Hide file tree

Showing 14 changed files with 54 additions and 47 deletions.
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
@@ -111,8 +111,9 @@ additional-extras:
       # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
       # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
       # In addition FAB also limit sqlalchemy to < 2.0
-      - pandas>=1.5.3,<2.2;python_version<"3.12"
-      - pandas>=2.1.1,<2.2;python_version>="3.12"
+      - pandas>=2.1.2,<2.2;python_version>="3.9"
+      - pandas>=1.5.3,<2.2;python_version<"3.9"
+
 
   # There is conflict between boto3 and aiobotocore dependency botocore.
   # TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or

diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml
@@ -55,8 +55,8 @@ dependencies:
   - apache-airflow>=2.7.0
   - hdfs[avro,dataframe,kerberos]>=2.5.4;python_version<"3.12"
   - hdfs[avro,dataframe,kerberos]>=2.7.3;python_version>="3.12"
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 
 integrations:

diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml
@@ -77,8 +77,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
   - pyhive[hive_pure_sasl]>=0.7.0
   - thrift>=0.11.0

diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml
@@ -66,8 +66,8 @@ additional-extras:
       # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
       # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
       # In addition FAB also limit sqlalchemy to < 2.0
-      - pandas>=1.5.3,<2.2;python_version<"3.12"
-      - pandas>=2.1.1,<2.2;python_version>="3.12"
+      - pandas>=2.1.2,<2.2;python_version>="3.9"
+      - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 
 integrations:

diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml
@@ -73,8 +73,8 @@ dependencies:
   - databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0
   - aiohttp>=3.9.2, <4
   - mergedeep>=1.3.4
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
   - pyarrow>=14.0.1
 
 additional-extras:

diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml
@@ -64,8 +64,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 
 integrations:

diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
@@ -162,8 +162,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2.0;python_version<"3.12"
-  - pandas>=2.1.1,<2.2.0;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
   # A transient dependency of google-cloud-bigquery-datatransfer, but we
   # further constrain it since older versions are buggy.
   - proto-plus>=1.19.6

diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml
@@ -58,8 +58,8 @@ dependencies:
   - papermill[all]>=2.4.0
   - scrapbook[all]
   - ipykernel
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 
 integrations:

diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml
@@ -66,8 +66,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 
 integrations:

diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml
@@ -62,8 +62,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 
 integrations:

diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml
@@ -75,9 +75,15 @@ versions:
 dependencies:
   - apache-airflow>=2.7.0
   - apache-airflow-providers-common-sql>=1.10.0
+  # In pandas 2.2 minimal version of the sqlalchemy is 2.0
+  # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
+  # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
+  # In addition FAB also limit sqlalchemy to < 2.0
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
+  - pyarrow>=14.0.1
   - snowflake-connector-python>=3.7.1
   - snowflake-sqlalchemy>=1.4.0
-  - pyarrow>=14.0.1
 
 integrations:
   - integration-name: Snowflake

diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml
@@ -67,9 +67,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
-
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
   - trino>=0.318.0
 
 integrations:

diff --git a/airflow/providers/weaviate/provider.yaml b/airflow/providers/weaviate/provider.yaml
@@ -54,9 +54,8 @@ dependencies:
   # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
   # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
   # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=1.5.3,<2.2;python_version<"3.12"
-  - pandas>=2.1.1,<2.2;python_version>="3.12"
-
+  - pandas>=2.1.2,<2.2;python_version>="3.9"
+  - pandas>=1.5.3,<2.2;python_version<"3.9"
 
 hooks:
   - integration-name: Weaviate

diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
@@ -147,8 +147,8 @@
       "apache-airflow>=2.7.0",
       "hdfs[avro,dataframe,kerberos]>=2.5.4;python_version<\"3.12\"",
       "hdfs[avro,dataframe,kerberos]>=2.7.3;python_version>=\"3.12\"",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\""
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\""
     ],
     "devel-deps": [],
     "plugins": [],
@@ -162,8 +162,8 @@
       "apache-airflow>=2.7.0",
       "hmsclient>=0.1.0",
       "jmespath>=0.7.0",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "pyhive[hive_pure_sasl]>=0.7.0",
       "thrift>=0.11.0"
     ],
@@ -417,8 +417,8 @@
       "apache-airflow>=2.7.0",
       "databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0",
       "mergedeep>=1.3.4",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "pyarrow>=14.0.1",
       "requests>=2.27.0,<3"
     ],
@@ -515,8 +515,8 @@
     "deps": [
       "apache-airflow-providers-common-sql>=1.10.0",
       "apache-airflow>=2.7.0",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "pyexasol>=0.5.1"
     ],
     "devel-deps": [],
@@ -635,8 +635,8 @@
       "json-merge-patch>=0.2",
       "looker-sdk>=22.4.0",
       "pandas-gbq>=0.7.0",
-      "pandas>=1.5.3,<2.2.0;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2.0;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "proto-plus>=1.19.6",
       "python-slugify>=7.0.0",
       "sqlalchemy-bigquery>=1.2.1",
@@ -980,8 +980,8 @@
     "deps": [
       "apache-airflow>=2.7.0",
       "ipykernel",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "papermill[all]>=2.4.0",
       "scrapbook[all]"
     ],
@@ -1039,8 +1039,8 @@
     "deps": [
       "apache-airflow-providers-common-sql>=1.3.1",
       "apache-airflow>=2.7.0",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "presto-python-client>=0.8.4"
     ],
     "devel-deps": [],
@@ -1077,8 +1077,8 @@
   "salesforce": {
     "deps": [
       "apache-airflow>=2.7.0",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "simple-salesforce>=1.0.0"
     ],
     "devel-deps": [],
@@ -1177,6 +1177,8 @@
     "deps": [
       "apache-airflow-providers-common-sql>=1.10.0",
       "apache-airflow>=2.7.0",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "pyarrow>=14.0.1",
       "snowflake-connector-python>=3.7.1",
       "snowflake-sqlalchemy>=1.4.0"
@@ -1273,8 +1275,8 @@
     "deps": [
       "apache-airflow-providers-common-sql>=1.3.1",
       "apache-airflow>=2.7.0",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "trino>=0.318.0"
     ],
     "devel-deps": [],
@@ -1305,8 +1307,8 @@
     "deps": [
       "apache-airflow>=2.7.0",
       "httpx>=0.25.0",
-      "pandas>=1.5.3,<2.2;python_version<\"3.12\"",
-      "pandas>=2.1.1,<2.2;python_version>=\"3.12\"",
+      "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
+      "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
       "weaviate-client>=3.24.2"
     ],
     "devel-deps": [],