Skip to content

Commit

Permalink
airbyte-ci: ignore archived connectors (airbytehq#43426)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alafanechere @LouisAuneau
alafanechere authored and LouisAuneau committed Aug 13, 2024
1 parent 24a405d commit 77ac2db
Show file tree
Hide file tree
Showing 6 changed files with 20 additions and 6 deletions.
3 changes: 2 additions & 1 deletion airbyte-ci/connectors/pipelines/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ Available commands:
| `--metadata-query` | False | | | Filter connectors by the `data` field in the metadata file using a [simpleeval](https://github.com/danthedeckie/simpleeval) query. e.g. 'data.ab_internal.ql == 200' |
| `--use-local-cdk` | False | False | | Build with the airbyte-cdk from the local repository. " "This is useful for testing changes to the CDK. |
| `--language` | True | | | Select connectors with a specific language: `python`, `low-code`, `java`. Can be used multiple times to select multiple languages. |
| `--modified` | False | False | | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation). |
| `--modified` | False | False | | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation). Archived connectors are ignored. |
| `--concurrency` | False | 5 | | Control the number of connector pipelines that can run in parallel. Useful to speed up pipelines or control their resource usage. |
| `--metadata-change-only/--not-metadata-change-only` | False | `--not-metadata-change-only` | | Only run the pipeline on connectors with changes on their metadata.yaml file. |
| `--enable-dependency-scanning / --disable-dependency-scanning` | False | ` --disable-dependency-scanning` | | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change. |
Expand Down Expand Up @@ -789,6 +789,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only

| Version | PR | Description |
|---------| ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------|
| 4.31.3 | [#43426](https://github.com/airbytehq/airbyte/pull/43426) | Ignore archived connectors on connector selection from modified files. |
| 4.31.2 | [#43433](https://github.com/airbytehq/airbyte/pull/43433) | Fix 'changed_file' indentation in 'pull-request' command |
| 4.31.1 | [#43442](https://github.com/airbytehq/airbyte/pull/43442) | Resolve type check failure in bump version |
| 4.31.0 | [#42970](https://github.com/airbytehq/airbyte/pull/42970) | Add explicit version set to bump version |
Expand Down
7 changes: 6 additions & 1 deletion airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,12 @@ def validate_environment(is_local: bool) -> None:
help="Filter connectors to test by support_level.",
type=click.Choice(SupportLevelEnum),
)
@click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool)
@click.option(
"--modified/--not-modified",
help="Only test modified connectors in the current branch. Archived connectors are ignored",
default=False,
type=bool,
)
@click.option(
"--metadata-changes-only/--not-metadata-changes-only",
help="Only test connectors with modified metadata files in the current branch.",
Expand Down
3 changes: 3 additions & 0 deletions airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ def _find_modified_connectors(
modified_connectors = set()

for connector in all_connectors:
if connector.support_level == "archived":
main_logger.info(f"Skipping connector '{connector}' due to 'archived' support level.")
continue
if Path(file_path).is_relative_to(Path(connector.code_directory)) or file_path == connector.documentation_file_path:
main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.")
modified_connectors.add(connector)
Expand Down
2 changes: 1 addition & 1 deletion airbyte-ci/connectors/pipelines/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "pipelines"
version = "4.31.2"
version = "4.31.3"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte <contact@airbyte.io>"]

Expand Down
7 changes: 5 additions & 2 deletions airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,11 @@ def test_get_selected_connectors_with_modified_and_language():
metadata_query=None,
modified_files=modified_files,
)

assert len(selected_connectors) == 1
has_strict_encrypt_variant = any("-strict-encrypt" in c.technical_name for c in selected_connectors)
if has_strict_encrypt_variant:
assert len(selected_connectors) == 2
else:
assert len(selected_connectors) == 1
assert selected_connectors[0].technical_name == second_modified_connector.technical_name


Expand Down
4 changes: 3 additions & 1 deletion airbyte-ci/connectors/pipelines/tests/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ def pick_a_random_connector(
all_connectors = [c for c in all_connectors if c.language is language]
if support_level:
all_connectors = [c for c in all_connectors if c.support_level == support_level]
else:
all_connectors = [c for c in all_connectors if c.support_level != "archived"]
picked_connector = random.choice(all_connectors)
if other_picked_connectors:
while picked_connector in other_picked_connectors:
Expand All @@ -44,7 +46,7 @@ def pick_a_random_connector(

def pick_a_strict_encrypt_variant_pair():
for c in ALL_CONNECTORS:
if c.technical_name.endswith("-strict-encrypt"):
if c.technical_name.endswith("-strict-encrypt") and c.support_level != "archived":
main_connector = Connector(c.relative_connector_path.replace("-strict-encrypt", ""))
return main_connector, c

Expand Down

0 comments on commit 77ac2db

Please sign in to comment.