Migrate Amazon provider package

Fixes: apache#46045

.github/boring-cyborg.yml

@@ -25,10 +25,7 @@ labelPRBasedOnFilePath:
     - providers/alibaba/**
 
   provider:amazon:
-    - providers/src/airflow/providers/amazon/**/*
-    - providers/tests/amazon/aws/**/*
-    - docs/apache-airflow-providers-amazon/**/*
-    - providers/tests/system/amazon/aws/**/*
+    - providers/amazon/**
 
   provider:apache-beam:
     - providers/apache/beam/**
@@ -300,12 +297,6 @@ labelPRBasedOnFilePath:
   provider:zendesk:
     - providers/zendesk/**
 
-  area:providers:
-    - providers/src/airflow/providers/**/*
-    - docs/apache-airflow-providers-*/**/*
-    - providers/tests/**/*
-    - providers/tests/system/**/*
-
   area:API:
     - airflow/api/**/*
     - airflow/api_connexion/**/*
@@ -371,11 +362,10 @@ labelPRBasedOnFilePath:
     - docs/apache-airflow/administration-and-deployment/lineage.rst
 
   area:Logging:
-    - providers/src/airflow/providers/**/log/*
     - airflow/utils/log/**/*
     - docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-*.rst
-    - providers/tests/**/log/*
     - tests/utils/log/**/*
+    - providers/**/log/*
 
   area:Plugins:
     - airflow/cli/commands/local_commands/plugins_command.py
@@ -403,9 +393,8 @@ labelPRBasedOnFilePath:
 
   area:Secrets:
     - airflow/secrets/**/*
-    - providers/src/airflow/providers/**/secrets/*
     - tests/secrets/**/*
-    - providers/tests/**/secrets/*
+    - providers/**/secrets/*
     - docs/apache-airflow/security/secrets/**/*
 
   area:Triggerer:

.pre-commit-config.yaml

@@ -221,7 +221,7 @@ repos:
         entry: ./scripts/ci/pre_commit/check_deferrable_default.py
         pass_filenames: false
         additional_dependencies: ["libcst>=1.1.0"]
-        files: ^(providers/src/)?airflow/.*/(sensors|operators)/.*\.py$
+        files: ^(providers/.*/)?airflow/.*/(sensors|operators)/.*\.py$
   - repo: https://github.com/asottile/blacken-docs
     rev: 1.19.1
     hooks:
@@ -338,7 +338,7 @@ repos:
         language: python
         entry: ./scripts/ci/pre_commit/validate_operators_init.py
         pass_filenames: true
-        files: ^providers/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$
+        files: ^providers/.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$
         additional_dependencies: [ 'rich>=12.4.4' ]
       - id: update-providers-build-files
         name: Update providers build files
@@ -460,7 +460,7 @@ repos:
         description: Check dependency of SQL Providers with common data structure
         entry: ./scripts/ci/pre_commit/check_common_sql_dependency.py
         language: python
-        files: ^providers/src/airflow/providers/.*/hooks/.*\.py$
+        files: ^providers/.*/src/airflow/providers/.*/hooks/.*\.py$
         additional_dependencies: ['rich>=12.4.4', 'pyyaml', 'packaging']
       - id: update-providers-dependencies
         name: Update dependencies for provider packages
@@ -559,10 +559,10 @@ repos:
           .*https://github.*[0-9]/providers/tests/system/|
           .*https://github.*/main/providers/tests/system/|
           .*https://github.*/master/providers/tests/system/|
-          .*https://github.*/main/providers/src/airflow/providers/.*/example_dags/|
-          .*https://github.*/master/providers/src/airflow/providers/.*/example_dags/
+          .*https://github.*/main/providers/.*/src/airflow/providers/.*/example_dags/|
+          .*https://github.*/master/providers/.*/src/airflow/providers/.*/example_dags/
         pass_filenames: true
-        files: ^docs/apache-airflow-providers-.*\.rst
+        files: ^providers/.*/docs/.*\.rst
       - id: check-safe-filter-usage-in-html
         language: pygrep
         name: Don't use safe in templates
@@ -625,8 +625,8 @@ repos:
           ^airflow/operators/__init__.py$|
           ^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute.py$|
           ^providers/fab/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
-          ^providers/src/airflow/providers/amazon/aws/hooks/emr.py$|
-          ^providers/src/airflow/providers/amazon/aws/operators/emr.py$|
+          ^providers/amazon/src/airflow/providers/amazon/aws/hooks/emr.py$|
+          ^providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py$|
           ^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra.py$|
           ^providers/apache/hive/src/airflow/providers/apache/hive/operators/hive_stats.py$|
           ^providers/apache/hive/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py$|
@@ -802,7 +802,7 @@ repos:
         name: Check if aiobotocore is an optional dependency only
         entry: ./scripts/ci/pre_commit/check_aiobotocore_optional.py
         language: python
-        files: ^providers/src/airflow/providers/.*/provider\.yaml$
+        files: ^providers/.*/provider\.yaml$
         pass_filenames: true
         additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml']
         require_serial: true
@@ -1112,13 +1112,13 @@ repos:
       - id: check-changelog-has-no-duplicates
         name: Check changelogs for duplicate entries
         language: python
-        files: CHANGELOG\.(rst|txt)$
+        files: changelog\.(rst|txt)$
         entry: ./scripts/ci/pre_commit/changelog_duplicates.py
         pass_filenames: true
       - id: check-changelog-format
         name: Check changelog format
         language: python
-        files: CHANGELOG\.(rst|txt)$
+        files: changelog\.(rst|txt)$
         entry: ./scripts/ci/pre_commit/check_changelog_format.py
         pass_filenames: true
       - id: check-newsfragments-are-valid
@@ -1168,7 +1168,7 @@ repos:
         entry: ./scripts/ci/pre_commit/check_system_tests_hidden_in_index.py
         language: python
         pass_filenames: true
-        files: ^docs/apache-airflow-providers-[^/]*/index\.rst$
+        files: ^providers/.*/docs/index\.rst$
         additional_dependencies: ['rich>=12.4.4', 'pyyaml']
       - id: check-lazy-logging
         name: Check that all logging methods are lazy
@@ -1415,13 +1415,14 @@ repos:
         entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py
         language: python
         files: ^providers/.*/provider\.yaml$
+        exclude: ^providers/.*/.venv/.*$
         additional_dependencies: ['rich>=12.4.4']
         require_serial: true
       - id: check-template-fields-valid
         name: Check templated fields mapped in operators/sensors
         language: python
         entry: ./scripts/ci/pre_commit/check_template_fields.py
-        files: ^(providers/src/)?airflow/.*/(sensors|operators)/.*\.py$
+        files: ^(providers/.*/)?airflow/.*/(sensors|operators)/.*\.py$
         additional_dependencies: [ 'rich>=12.4.4' ]
         require_serial: true
       - id: update-migration-references

dev/breeze/tests/test_packages.py

@@ -37,7 +37,6 @@
     get_old_source_providers_package_path,
     get_pip_package_name,
     get_provider_info_dict,
-    get_provider_jinja_context,
     get_provider_requirements,
     get_removed_provider_ids,
     get_short_package_name,
@@ -379,38 +378,5 @@ def test_get_provider_info_dict():
     assert len(provider_info_dict["notifications"]) > 2
     assert len(provider_info_dict["secrets-backends"]) > 1
     assert len(provider_info_dict["logging"]) > 1
-    assert len(provider_info_dict["additional-extras"]) > 3
     assert len(provider_info_dict["config"].keys()) > 1
     assert len(provider_info_dict["executors"]) > 0
-
-
-# TODO(potiuk) - remove when all providers are new-style
-def test_old_provider_jinja_context():
-    provider_info = get_provider_info_dict("amazon")
-    version = provider_info["versions"][0]
-    context = get_provider_jinja_context(
-        provider_id="amazon", current_release_version=version, version_suffix="rc1"
-    )
-    expected = {
-        "PROVIDER_ID": "amazon",
-        "PACKAGE_PIP_NAME": "apache-airflow-providers-amazon",
-        "PACKAGE_DIST_PREFIX": "apache_airflow_providers_amazon",
-        "FULL_PACKAGE_NAME": "airflow.providers.amazon",
-        "RELEASE": version,
-        "RELEASE_NO_LEADING_ZEROS": version,
-        "VERSION_SUFFIX": ".rc1",
-        "PROVIDER_DESCRIPTION": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
-        "CHANGELOG_RELATIVE_PATH": "../../providers/src/airflow/providers/amazon",
-        "SUPPORTED_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"],
-        "PLUGINS": [],
-        "MIN_AIRFLOW_VERSION": "2.9.0",
-        "PROVIDER_REMOVED": False,
-        "PROVIDER_INFO": provider_info,
-    }
-
-    for key, value in expected.items():
-        assert context[key] == value
-    assert """"google" = [
-     "apache-airflow-providers-google",
-]""" in context["EXTRAS_REQUIREMENTS"]
-    assert len(context["PIP_REQUIREMENTS"]) > 10

dev/breeze/tests/test_pytest_args_for_test_types.py

@@ -88,7 +88,7 @@ def _all_new_providers() -> list[str]:
         (
             GroupOfTests.PROVIDERS,
             "Providers[amazon]",
-            ["providers/tests/amazon"],
+            ["providers/amazon/tests"],
         ),
         (
             GroupOfTests.PROVIDERS,
@@ -99,7 +99,7 @@ def _all_new_providers() -> list[str]:
             GroupOfTests.PROVIDERS,
             "Providers[amazon,google,apache.hive]",
             [
-                "providers/tests/amazon",
+                "providers/amazon/tests",
                 "providers/google/tests",
                 "providers/apache/hive/tests",
             ],
@@ -114,7 +114,6 @@ def _all_new_providers() -> list[str]:
                     if provider not in ["amazon", "google", "microsoft/azure"]
                 ],
                 "providers/tests",
-                "--ignore=providers/tests/amazon",
             ],
         ),
         (
@@ -245,14 +244,14 @@ def test_pytest_args_for_missing_provider():
             GroupOfTests.PROVIDERS,
             "Providers[amazon]",
             [
-                "providers/tests/amazon",
+                "providers/amazon/tests",
             ],
         ),
         (
             GroupOfTests.PROVIDERS,
             "Providers[amazon] Providers[google]",
             [
-                "providers/tests/amazon",
+                "providers/amazon/tests",
                 "providers/google/tests",
             ],
         ),
@@ -278,9 +277,7 @@ def test_pytest_args_for_missing_provider():
                     if provider not in ["amazon", "google"]
                 ],
                 "providers/tests",
-                *[
-                    "providers/google/tests"
-                ],  # Once amazon is migrated to the new structure, amazon needs to be added to the list here.
+                *["providers/amazon/tests", "providers/google/tests"],
             ],
         ),
         (

dev/breeze/tests/test_selective_checks.py

@@ -717,7 +717,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             )
         ),
         pytest.param(
-            ("providers/src/airflow/providers/amazon/__init__.py",),
+            ("providers/amazon/src/airflow/providers/amazon/__init__.py",),
             {
                 "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
                 "common.compat common.sql exasol ftp google http imap microsoft.azure "
@@ -769,7 +769,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             id="Providers tests run without amazon tests if no amazon file changed",
         ),
         pytest.param(
-            ("providers/src/airflow/providers/amazon/file.py",),
+            ("providers/amazon/src/airflow/providers/amazon/file.py",),
             {
                 "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
                 "common.compat common.sql exasol ftp google http imap microsoft.azure "

docs/.gitignore

@@ -2,6 +2,7 @@
 # Eventually when we swtich to individually build docs for each provider, we should remove this altogether
 apache-airflow-providers-airbyte
 apache-airflow-providers-alibaba
+apache-airflow-providers-amazon
 apache-airflow-providers-apache-beam
 apache-airflow-providers-apache-cassandra
 apache-airflow-providers-apache-drill