From bc6bd50d3978f23b23e27fc633cb11bb12384d38 Mon Sep 17 00:00:00 2001 From: Augustin Date: Mon, 12 Aug 2024 11:20:11 +0200 Subject: [PATCH] airbyte-ci: ignore archived connectors (#43426) --- airbyte-ci/connectors/pipelines/README.md | 3 ++- .../pipelines/pipelines/airbyte_ci/connectors/commands.py | 7 ++++++- .../pipelines/pipelines/helpers/connectors/modifed.py | 3 +++ airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- .../tests/test_commands/test_groups/test_connectors.py | 7 +++++-- airbyte-ci/connectors/pipelines/tests/utils.py | 4 +++- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index e569a9639710..0e811f32d94a 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -253,7 +253,7 @@ Available commands: | `--metadata-query` | False | | | Filter connectors by the `data` field in the metadata file using a [simpleeval](https://github.com/danthedeckie/simpleeval) query. e.g. 'data.ab_internal.ql == 200' | | `--use-local-cdk` | False | False | | Build with the airbyte-cdk from the local repository. " "This is useful for testing changes to the CDK. | | `--language` | True | | | Select connectors with a specific language: `python`, `low-code`, `java`. Can be used multiple times to select multiple languages. | -| `--modified` | False | False | | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation). | +| `--modified` | False | False | | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation). Archived connectors are ignored. | | `--concurrency` | False | 5 | | Control the number of connector pipelines that can run in parallel. Useful to speed up pipelines or control their resource usage. | | `--metadata-change-only/--not-metadata-change-only` | False | `--not-metadata-change-only` | | Only run the pipeline on connectors with changes on their metadata.yaml file. | | `--enable-dependency-scanning / --disable-dependency-scanning` | False | ` --disable-dependency-scanning` | | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change. | @@ -789,6 +789,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only | Version | PR | Description | |---------| ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------| +| 4.31.3 | [#43426](https://github.com/airbytehq/airbyte/pull/43426) | Ignore archived connectors on connector selection from modified files. | | 4.31.2 | [#43433](https://github.com/airbytehq/airbyte/pull/43433) | Fix 'changed_file' indentation in 'pull-request' command | | 4.31.1 | [#43442](https://github.com/airbytehq/airbyte/pull/43442) | Resolve type check failure in bump version | | 4.31.0 | [#42970](https://github.com/airbytehq/airbyte/pull/42970) | Add explicit version set to bump version | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 775a296e3509..172b639bb968 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -169,7 +169,12 @@ def validate_environment(is_local: bool) -> None: help="Filter connectors to test by support_level.", type=click.Choice(SupportLevelEnum), ) -@click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool) +@click.option( + "--modified/--not-modified", + help="Only test modified connectors in the current branch. Archived connectors are ignored", + default=False, + type=bool, +) @click.option( "--metadata-changes-only/--not-metadata-changes-only", help="Only test connectors with modified metadata files in the current branch.", diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py index 718cc283f8c6..3bc7b0812a67 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py @@ -27,6 +27,9 @@ def _find_modified_connectors( modified_connectors = set() for connector in all_connectors: + if connector.support_level == "archived": + main_logger.info(f"Skipping connector '{connector}' due to 'archived' support level.") + continue if Path(file_path).is_relative_to(Path(connector.code_directory)) or file_path == connector.documentation_file_path: main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.") modified_connectors.add(connector) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index d223c3f93e9c..440bcc7438f8 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.31.2" +version = "4.31.3" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py index 88632147357f..bd0304d744d6 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -133,8 +133,11 @@ def test_get_selected_connectors_with_modified_and_language(): metadata_query=None, modified_files=modified_files, ) - - assert len(selected_connectors) == 1 + has_strict_encrypt_variant = any("-strict-encrypt" in c.technical_name for c in selected_connectors) + if has_strict_encrypt_variant: + assert len(selected_connectors) == 2 + else: + assert len(selected_connectors) == 1 assert selected_connectors[0].technical_name == second_modified_connector.technical_name diff --git a/airbyte-ci/connectors/pipelines/tests/utils.py b/airbyte-ci/connectors/pipelines/tests/utils.py index 9f6791ccc970..958858d72ffb 100644 --- a/airbyte-ci/connectors/pipelines/tests/utils.py +++ b/airbyte-ci/connectors/pipelines/tests/utils.py @@ -35,6 +35,8 @@ def pick_a_random_connector( all_connectors = [c for c in all_connectors if c.language is language] if support_level: all_connectors = [c for c in all_connectors if c.support_level == support_level] + else: + all_connectors = [c for c in all_connectors if c.support_level != "archived"] picked_connector = random.choice(all_connectors) if other_picked_connectors: while picked_connector in other_picked_connectors: @@ -44,7 +46,7 @@ def pick_a_random_connector( def pick_a_strict_encrypt_variant_pair(): for c in ALL_CONNECTORS: - if c.technical_name.endswith("-strict-encrypt"): + if c.technical_name.endswith("-strict-encrypt") and c.support_level != "archived": main_connector = Connector(c.relative_connector_path.replace("-strict-encrypt", "")) return main_connector, c