Move Hive macros to the provider

airflow/macros/__init__.py

@@ -26,7 +26,16 @@
 import dateutil  # noqa
 from pendulum import DateTime
 
-from airflow.macros import hive  # noqa
+from airflow.utils.deprecation_tools import add_deprecated_classes
+
+__deprecated_classes = {
+    "hive": {
+        "closest_ds_partition": "airflow.providers.apache.hive.macros.hive.closest_ds_partition",
+        "max_partition": "airflow.providers.apache.hive.macros.hive.max_partition",
+    },
+}
+
+add_deprecated_classes(__deprecated_classes, __name__)
 
 
 def ds_add(ds: str, days: int) -> str:

airflow/provider.yaml.schema.json

@@ -276,6 +276,18 @@
       "items": {
           "type": "string"
       }
+    },
+    "plugins": {
+      "type": "array",
+      "description": "Plugins exposed by the provider",
+      "items": {
+        "name": {
+           "type": "string"
+        },
+        "plugin-class": {
+           "type": "string"
+        }
+      }
     }
   },
   "additionalProperties": false,

airflow/providers/apache/hive/CHANGELOG.rst

@@ -24,6 +24,15 @@
 Changelog
 ---------
 
+5.1.0
+.....
+
+Features
+~~~~~~~~
+
+The ``apache.hive`` provider provides now hive macros that used to be provided by Airflow. As of 5.1.0 version
+of ``apache.hive`` the hive macros are provided by the Provider.
+
 5.0.0
 .....
 

airflow/providers/apache/hive/macros/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/apache/hive/plugins/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/apache/hive/plugins/hive.py

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.providers.apache.hive.macros.hive import closest_ds_partition, max_partition
+
+
+class HivePlugin(AirflowPlugin):
+    """Hive plugin - delivering macros used by users that use the provider."""
+
+    name = "hive"
+    macros = [max_partition, closest_ds_partition]

airflow/providers/apache/hive/provider.yaml

@@ -22,6 +22,7 @@ description: |
   `Apache Hive <https://hive.apache.org/>`__
 
 versions:
+  - 5.1.0
   - 5.0.0
   - 4.1.1
   - 4.1.0
@@ -109,3 +110,7 @@ connection-types:
     connection-type: hiveserver2
   - hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook
     connection-type: hive_metastore
+
+plugins:
+  - name: hive
+    plugin-class: airflow.providers.apache.hive.plugins.hive.HivePlugin

airflow/providers/apache/hive/sensors/metastore_partition.py

@@ -31,7 +31,7 @@ class MetastorePartitionSensor(SqlSensor):
     MySQL db. This was created as a result of observing sub optimal
     queries generated by the Metastore thrift service when hitting
     subpartitioned tables. The Thrift service's queries were written in a
-    way that wouldn't leverage the indexes.
+    way that would not leverage the indexes.
 
     :param schema: the schema
     :param table: the table

dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2

@@ -71,6 +71,13 @@ install_requires = {{ INSTALL_REQUIREMENTS }}
 [options.entry_points]
 apache_airflow_provider=
     provider_info=airflow.providers.{{ PROVIDER_PACKAGE_ID }}.get_provider_info:get_provider_info
+{%- if PLUGINS %}
+airflow.plugins=
+{%- for plugin in PLUGINS %}
+    {{ plugin.name }}={{ plugin.package_name }}:{{ plugin.class_name }}
+{%- endfor %}
+{%- endif %}
+
 
 [files]
 packages = airflow.providers.{{ PROVIDER_PACKAGE_ID }}

dev/provider_packages/prepare_provider_packages.py

@@ -124,6 +124,12 @@
 console = Console(width=400, color_system="standard")
 
 
+class PluginInfo(NamedTuple):
+    name: str
+    package_name: str
+    class_name: str
+
+
 class ProviderPackageDetails(NamedTuple):
     provider_package_id: str
     full_package_name: str
@@ -133,6 +139,7 @@ class ProviderPackageDetails(NamedTuple):
     provider_description: str
     versions: list[str]
     excluded_python_versions: list[str]
+    plugins: list[PluginInfo]
 
 
 class EntityType(Enum):
@@ -1014,6 +1021,17 @@ def get_all_changes_for_package(
 
 def get_provider_details(provider_package_id: str) -> ProviderPackageDetails:
     provider_info = get_provider_info_from_provider_yaml(provider_package_id)
+    plugins: list[PluginInfo] = []
+    if "plugins" in provider_info:
+        for plugin in provider_info["plugins"]:
+            package_name, class_name = plugin["plugin-class"].rsplit(".", maxsplit=1)
+            plugins.append(
+                PluginInfo(
+                    name=plugin["name"],
+                    package_name=package_name,
+                    class_name=class_name,
+                )
+            )
     return ProviderPackageDetails(
         provider_package_id=provider_package_id,
         full_package_name=f"airflow.providers.{provider_package_id}",
@@ -1023,6 +1041,7 @@ def get_provider_details(provider_package_id: str) -> ProviderPackageDetails:
         provider_description=provider_info["description"],
         versions=provider_info["versions"],
         excluded_python_versions=provider_info.get("excluded-python-versions") or [],
+        plugins=plugins,
     )
 
 
@@ -1099,6 +1118,7 @@ def get_provider_jinja_context(
         "CHANGELOG": changelog,
         "SUPPORTED_PYTHON_VERSIONS": supported_python_versions,
         "PYTHON_REQUIRES": python_requires,
+        "PLUGINS": provider_details.plugins,
     }
     return context
 

docs/apache-airflow-providers-apache-hive/index.rst

@@ -45,9 +45,10 @@ Content
     :maxdepth: 1
     :caption: Resources
 
-    Example DAGs <https://github.com/apache/airflow/tree/providers-apache-hive/|versrion|/tests/system/providers/apache/hive>
+    Example DAGs <https://github.com/apache/airflow/tree/providers-apache-hive/|version|/tests/system/providers/apache/hive>
     PyPI Repository <https://pypi.org/project/apache-airflow-providers-apache-hive/>
     Installing from sources <installing-providers-from-sources>
+    Macros <macros>
 
 .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
 

docs/apache-airflow-providers-apache-hive/macros.rst

@@ -0,0 +1,26 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Hive Macros
+===========
+
+The following macros are available to use in Jinja2 templates. You need to prefix them with ``hive`` when
+you use them in your templates - for example ``hive.closest_ds_partition``.
+
+.. automodule:: airflow.providers.apache.hive.macros.hive
+    :members:
+    :noindex:

docs/apache-airflow/templates-ref.rst

@@ -178,7 +178,4 @@ Some airflow specific macros are also defined:
 .. automodule:: airflow.macros
     :members:
 
-.. automodule:: airflow.macros.hive
-    :members:
-
 .. _pendulum.DateTime: https://pendulum.eustace.io/docs/#introduction

newsfragments/28538.misc.rst

@@ -0,0 +1 @@
+The Hive Macros (``hive.max_partition``, ``hive.closest_ds_partition``) are available only when Hive Provider is installed. Please install Hive Provider > 5.1.0 when using those macros.

scripts/in_container/run_provider_yaml_files_check.py

@@ -293,6 +293,22 @@ def check_hook_classes(yaml_files: dict[str, dict]):
             )
 
 
+def check_plugin_classes(yaml_files: dict[str, dict]):
+    print("Checking plugin classes belong to package, exist and are classes")
+    resource_type = "plugins"
+    for yaml_file_path, provider_data in yaml_files.items():
+        provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".")
+        plugins = provider_data.get(resource_type)
+        if plugins:
+            check_if_objects_exist_and_belong_to_package(
+                {plugin["plugin-class"] for plugin in plugins},
+                provider_package,
+                yaml_file_path,
+                resource_type,
+                ObjectType.CLASS,
+            )
+
+
 def check_extra_link_classes(yaml_files: dict[str, dict]):
     print("Checking extra-links belong to package, exist and are classes")
     resource_type = "extra-links"
@@ -464,6 +480,7 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]):
     check_completeness_of_list_of_transfers(all_parsed_yaml_files)
     check_duplicates_in_list_of_transfers(all_parsed_yaml_files)
     check_hook_classes(all_parsed_yaml_files)
+    check_plugin_classes(all_parsed_yaml_files)
     check_extra_link_classes(all_parsed_yaml_files)
     check_correctness_of_list_of_sensors_operators_hook_modules(all_parsed_yaml_files)
     check_unique_provider_name(all_parsed_yaml_files)

setup.py

@@ -810,6 +810,10 @@ def replace_extra_dependencies_with_provider_packages(extra: str, providers: lis
         EXTRAS_DEPENDENCIES[extra].extend(
             [get_provider_package_name_from_package_id(package_name) for package_name in providers]
         )
+    elif extra == "apache.hive":
+        # We moved the hive macros to the hive provider, and they are available in hive provider only as of
+        # 5.1.0 version only, so we have to make sure minimum version is used
+        EXTRAS_DEPENDENCIES[extra] = ["apache-airflow-providers-hive>=5.1.0"]
     else:
         EXTRAS_DEPENDENCIES[extra] = [
             get_provider_package_name_from_package_id(package_name) for package_name in providers

tests/providers/apache/hive/macros/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/macros/test_hive.py → tests/providers/apache/hive/macros/test_hive.py

@@ -19,7 +19,7 @@
 
 from datetime import datetime
 
-from airflow.macros import hive
+from airflow.providers.apache.hive.macros import hive
 
 
 class TestHive: