Remove snakebite-py3 based HDFS hooks and sensors

Those Hooks were using technology that is not going to work in
a new environment, with newer versions of Python supported,
protobufs that reached end of life.

We should finally get rid of them. Users can still use older
versions of providers with those old HDFS hooks and sensors if
needed.

airflow/contrib/sensors/__init__.py

@@ -114,12 +114,6 @@
             'airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionCompleteSensor'
         ),
     },
-    'hdfs_sensor': {
-        'HdfsFolderSensor': 'airflow.providers.apache.hdfs.sensors.hdfs.HdfsFolderSensor',
-        'HdfsRegexSensor': 'airflow.providers.apache.hdfs.sensors.hdfs.HdfsRegexSensor',
-        'HdfsSensorFolder': 'airflow.providers.apache.hdfs.sensors.hdfs.HdfsFolderSensor',
-        'HdfsSensorRegex': 'airflow.providers.apache.hdfs.sensors.hdfs.HdfsRegexSensor',
-    },
     'imap_attachment_sensor': {
         'ImapAttachmentSensor': 'airflow.providers.imap.sensors.imap_attachment.ImapAttachmentSensor',
     },

airflow/hooks/__init__.py

@@ -39,10 +39,6 @@
         "DruidDbApiHook": "airflow.providers.apache.druid.hooks.druid.DruidDbApiHook",
         "DruidHook": "airflow.providers.apache.druid.hooks.druid.DruidHook",
     },
-    "hdfs_hook": {
-        "HDFSHook": "airflow.providers.apache.hdfs.hooks.hdfs.HDFSHook",
-        "HDFSHookException": "airflow.providers.apache.hdfs.hooks.hdfs.HDFSHookException",
-    },
     "hive_hooks": {
         "HIVE_QUEUE_PRIORITIES": "airflow.providers.apache.hive.hooks.hive.HIVE_QUEUE_PRIORITIES",
         "HiveCliHook": "airflow.providers.apache.hive.hooks.hive.HiveCliHook",

airflow/providers/apache/hdfs/CHANGELOG.rst

@@ -24,6 +24,24 @@
 Changelog
 ---------
 
+4.0.0
+-----
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+The original HDFS Hook and sensor has been removed. It used the old HDFS snakebite-py3 library that had no
+update in years and the protobuf they are using reached end of life.
+
+The 3.* version of the provider is still available and can be used if you need to use the old hooks and
+sensors.
+
+The ``HDFSHook``, ``HDFSSensor``, ``HdfsRegexSensor``, ``HdfsRegexSensor`` that have been removed from
+this provider and they are not available any more. If you want to continue using them,
+you can use 3.* version of the provider, but the recommendation is to switch to the new
+``WebHDFSHook`` and ``WebHDFSSensor`` that use the ``WebHDFS`` API.
+
+
 3.2.1
 .....
 

airflow/providers/apache/hdfs/hooks/hdfs.py

@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,98 +14,38 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Hook for HDFS operations"""
 from __future__ import annotations
 
-from typing import Any
-
-from airflow.configuration import conf
-from airflow.exceptions import AirflowException
+from airflow import AirflowException
 from airflow.hooks.base import BaseHook
 
-try:
-    from snakebite.client import AutoConfigClient, Client, HAClient, Namenode
-
-    snakebite_loaded = True
-except ImportError:
-    snakebite_loaded = False
+_EXCEPTION_MESSAGE = """The old HDFS Hooks have been removed in 4.0.0 version of the apache.hdfs provider.
+Please convert your DAGs to use the WebHdfsHook or downgrade the provider to below 4.*
+if you want to continue using it.
+If you want to use earlier provider you can downgrade to latest released 3.* version
+using `pip install apache-airflow-providers-hdfs==3.2.1` (no constraints)
+"""
 
 
 class HDFSHookException(AirflowException):
-    """Exception specific for HDFS"""
-
-
-class HDFSHook(BaseHook):
     """
-    Interact with HDFS. This class is a wrapper around the snakebite library.
-
-    :param hdfs_conn_id: Connection id to fetch connection info
-    :param proxy_user: effective user for HDFS operations
-    :param autoconfig: use snakebite's automatically configured client
+    This Exception has been removed and is not functional. Please convert your DAGs to use the
+    WebHdfsHook or downgrade the provider to below 4.* if you want to continue using it.
+    If you want to use earlier provider you can downgrade to latest released 3.* version
+    using `pip install apache-airflow-providers-hdfs==3.2.1` (no constraints).
     """
 
-    conn_name_attr = "hdfs_conn_id"
-    default_conn_name = "hdfs_default"
-    conn_type = "hdfs"
-    hook_name = "HDFS"
+    def __init__(self, *args, **kwargs):
+        raise Exception(_EXCEPTION_MESSAGE)
 
-    def __init__(
-        self,
-        hdfs_conn_id: str | set[str] = "hdfs_default",
-        proxy_user: str | None = None,
-        autoconfig: bool = False,
-    ):
-        super().__init__()
-        if not snakebite_loaded:
-            raise ImportError(
-                "This HDFSHook implementation requires snakebite, but "
-                "snakebite is not compatible with Python 3 "
-                "(as of August 2015). Please help by submitting a PR!"
-            )
-        self.hdfs_conn_id = {hdfs_conn_id} if isinstance(hdfs_conn_id, str) else hdfs_conn_id
-        self.proxy_user = proxy_user
-        self.autoconfig = autoconfig
 
-    def get_conn(self) -> Any:
-        """Returns a snakebite HDFSClient object."""
-        # When using HAClient, proxy_user must be the same, so is ok to always
-        # take the first.
-        effective_user = self.proxy_user
-        autoconfig = self.autoconfig
-        use_sasl = conf.get("core", "security") == "kerberos"
-
-        try:
-            connections = [self.get_connection(i) for i in self.hdfs_conn_id]
-
-            if not effective_user:
-                effective_user = connections[0].login
-            if not autoconfig:
-                autoconfig = connections[0].extra_dejson.get("autoconfig", False)
-            hdfs_namenode_principal = connections[0].extra_dejson.get("hdfs_namenode_principal")
-        except AirflowException:
-            if not autoconfig:
-                raise
-
-        if autoconfig:
-            # will read config info from $HADOOP_HOME conf files
-            client = AutoConfigClient(effective_user=effective_user, use_sasl=use_sasl)
-        elif len(connections) == 1:
-            client = Client(
-                connections[0].host,
-                connections[0].port,
-                effective_user=effective_user,
-                use_sasl=use_sasl,
-                hdfs_namenode_principal=hdfs_namenode_principal,
-            )
-        elif len(connections) > 1:
-            name_node = [Namenode(conn.host, conn.port) for conn in connections]
-            client = HAClient(
-                name_node,
-                effective_user=effective_user,
-                use_sasl=use_sasl,
-                hdfs_namenode_principal=hdfs_namenode_principal,
-            )
-        else:
-            raise HDFSHookException("conn_id doesn't exist in the repository and autoconfig is not specified")
+class HDFSHook(BaseHook):
+    """
+    This Hook has been removed and is not functional. Please convert your DAGs to use the
+    WebHdfsHook or downgrade the provider to below 4.*. if you want to continue using it.
+    If you want to use earlier provider you can downgrade to latest released 3.* version
+    using `pip install apache-airflow-providers-hdfs==3.2.1` (no constraints).
+    """
 
-        return client
+    def __init__(self, *args, **kwargs):
+        raise Exception(_EXCEPTION_MESSAGE)

airflow/providers/apache/hdfs/provider.yaml

@@ -24,6 +24,7 @@ description: |
 
 suspended: false
 versions:
+  - 4.0.0
   - 3.2.1
   - 3.2.0
   - 3.1.0
@@ -41,14 +42,11 @@ versions:
 
 dependencies:
   - apache-airflow>=2.4.0
-  - snakebite-py3
   - hdfs[avro,dataframe,kerberos]>=2.0.4
 
 integrations:
   - integration-name: Hadoop Distributed File System (HDFS)
     external-doc-url: https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html
-    how-to-guide:
-      - /docs/apache-airflow-providers-apache-hdfs/operators/hdfs.rst
     logo: /integration-logos/apache/hadoop.png
     tags: [apache]
   - integration-name: WebHDFS
@@ -59,22 +57,11 @@ integrations:
     tags: [apache]
 
 sensors:
-  - integration-name: Hadoop Distributed File System (HDFS)
-    python-modules:
-      - airflow.providers.apache.hdfs.sensors.hdfs
   - integration-name: WebHDFS
     python-modules:
       - airflow.providers.apache.hdfs.sensors.web_hdfs
 
 hooks:
-  - integration-name: Hadoop Distributed File System (HDFS)
-    python-modules:
-      - airflow.providers.apache.hdfs.hooks.hdfs
   - integration-name: WebHDFS
     python-modules:
       - airflow.providers.apache.hdfs.hooks.webhdfs
-
-
-connection-types:
-  - hook-class-name: airflow.providers.apache.hdfs.hooks.hdfs.HDFSHook
-    connection-type: hdfs