diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index c44ac588408e..b3a90e6f3ee0 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -380,6 +380,7 @@ This command runs the Python tests for a airbyte-ci poetry package. | Version | PR | Description | | ------- | --------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | | 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | | 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | | 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py index 77587150de5a..48d42551721c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/bases.py +++ b/airbyte-ci/connectors/pipelines/pipelines/bases.py @@ -38,7 +38,7 @@ @dataclass(frozen=True) class ConnectorWithModifiedFiles(Connector): - modified_files: Set[Path] = field(default_factory=list) + modified_files: Set[Path] = field(default_factory=frozenset) @property def has_metadata_change(self) -> bool: diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py index ad27738bd4f5..59a233942157 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py @@ -78,4 +78,4 @@ async def run_test(airbyte_ci_package_path: str) -> bool: except dagger.ExecError as e: logger.error("Tests failed") logger.error(e.stderr) - return False + sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/utils.py b/airbyte-ci/connectors/pipelines/pipelines/utils.py index 9c395f87b779..d80fe8d744ed 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/utils.py @@ -15,7 +15,7 @@ from glob import glob from io import TextIOWrapper from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, FrozenSet, List, Optional, Set, Tuple, Union import anyio import asyncer @@ -359,13 +359,13 @@ def get_modified_connectors(modified_files: Set[Path], all_connectors: Set[Conne return modified_connectors -def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> Set[Path]: +def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> FrozenSet[Path]: connector_modified_files = set() for modified_file in all_modified_files: modified_file_path = Path(modified_file) if modified_file_path.is_relative_to(connector.code_directory): connector_modified_files.add(modified_file) - return connector_modified_files + return frozenset(connector_modified_files) def get_modified_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]: diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index df4d077f0be9..82b5af5bdbc8 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "0.4.3" +version = "0.4.4" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 6e42d1b2d9ab..188fbd44de22 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -15,12 +15,12 @@ from tests.utils import ALL_CONNECTORS -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def anyio_backend(): return "asyncio" -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") async def dagger_client(): async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: yield client diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py new file mode 100644 index 000000000000..f6623121ae89 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from pathlib import Path + +import pytest +from pipelines import bases, gradle + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestGradleTask: + class DummyStep(gradle.GradleTask): + gradle_task_name = "dummyTask" + + async def _run(self) -> bases.StepResult: + return bases.StepResult(self, bases.StepStatus.SUCCESS) + + @pytest.fixture + def test_context(self, mocker, dagger_client): + return mocker.Mock( + secrets_to_mask=[], + dagger_client=dagger_client, + connector=bases.ConnectorWithModifiedFiles( + "source-postgres", frozenset({Path("airbyte-integrations/connectors/source-postgres/metadata.yaml")}) + ), + ) + + async def test_build_include(self, test_context): + step = self.DummyStep(test_context) + assert step.build_include diff --git a/airbyte-ci/connectors/pipelines/tests/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py index 5d640b055bc0..35984a1bcef3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_utils.py @@ -140,3 +140,28 @@ def test_get_modified_connectors_with_dependency_scanning(all_connectors, enable else: assert not_modified_java_connector not in modified_connectors assert modified_java_connector in modified_connectors + + +def test_get_connector_modified_files(): + connector = pick_a_random_connector() + other_connector = pick_a_random_connector(other_picked_connectors=[connector]) + + all_modified_files = { + connector.code_directory / "setup.py", + other_connector.code_directory / "README.md", + } + + result = utils.get_connector_modified_files(connector, all_modified_files) + assert result == frozenset({connector.code_directory / "setup.py"}) + + +def test_no_modified_files_in_connector_directory(): + connector = pick_a_random_connector() + other_connector = pick_a_random_connector(other_picked_connectors=[connector]) + + all_modified_files = { + other_connector.code_directory / "README.md", + } + + result = utils.get_connector_modified_files(connector, all_modified_files) + assert result == frozenset()