From 554d45a13836e0f1a68cfe3089a5f08e115fe879 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Mon, 1 Jul 2024 09:40:33 -0400 Subject: [PATCH] CATs: add connector code to setup container (#40613) --- .../connector-acceptance-test/CHANGELOG.md | 4 +++ .../connector_acceptance_test/conftest.py | 1 + .../utils/setup_and_teardown_runner.py | 36 ++++++++++++++----- .../connector-acceptance-test/pyproject.toml | 2 +- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index c5fa6e820878f..f19ad58db2dc6 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.8.1 + +Provide the ability for users to perform setup/teardown by building and running a container before each test. + ## 3.8.0 Add `TestDiscovery.test_primary_keys_data_type`, which validates that primary keys are not of type `array` or `object` in discovered catalog. diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 45570164861a6..8efe46467b9ba 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -212,6 +212,7 @@ async def setup_and_teardown( logging.info("Running setup") setup_teardown_container = await setup_and_teardown_runner.do_setup( dagger_client, + base_path, base_path / setup_teardown_dockerfile_config.setup_teardown_dockerfile_path, setup_teardown_dockerfile_config.setup_command, connector_config, diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py index 56523b0ed419b..0a5ed0eb52a92 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py @@ -4,6 +4,7 @@ import json import os +from glob import glob from pathlib import Path from typing import List @@ -17,16 +18,33 @@ async def _build_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container: workspace = ( dagger_client.container() - .with_directory("/tmp", dagger_client.host().directory(os.path.dirname(dockerfile_path))) - .with_workdir("/tmp") - .with_file("/tmp/Dockerfile.cat_setup_teardown", dagger_client.host().file(str(dockerfile_path.expanduser()))) - .directory("/tmp") + .with_mounted_directory("/tmp/setup_teardown_context", dagger_client.host().directory(os.path.dirname(dockerfile_path))) + .directory("/tmp/setup_teardown_context") ) return await dagger_client.container().build(context=workspace, dockerfile="Dockerfile.cat_setup_teardown") -async def _build_setup_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container: - return await _build_container(dagger_client, dockerfile_path) +async def _build_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path) -> dagger.Container: + container = await _build_container(dagger_client, dockerfile_path) + return container.with_mounted_directory( + "/connector", dagger_client.host().directory(str(connector_path), exclude=get_default_excluded_files()) + ) + + +def get_default_excluded_files() -> List[str]: + return ( + [".git"] + + glob("**/build", recursive=True) + + glob("**/.venv", recursive=True) + + glob("**/__pycache__", recursive=True) + + glob("**/*.egg-info", recursive=True) + + glob("**/.vscode", recursive=True) + + glob("**/.pytest_cache", recursive=True) + + glob("**/.eggs", recursive=True) + + glob("**/.mypy_cache", recursive=True) + + glob("**/.DS_Store", recursive=True) + + glob("**/.gradle", recursive=True) + ) async def _run_with_config(container: dagger.Container, command: List[str], config: SecretDict) -> dagger.Container: @@ -38,8 +56,10 @@ async def _run(container: dagger.Container, command: List[str]) -> dagger.Contai return await container.with_exec(command, skip_entrypoint=True) -async def do_setup(dagger_client: dagger.Client, dockerfile_path: Path, command: List[str], connector_config: SecretDict): - return await _run_with_config(await _build_setup_container(dagger_client, dockerfile_path), command, connector_config) +async def do_setup( + dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path, command: List[str], connector_config: SecretDict +): + return await _run_with_config(await _build_client_container(dagger_client, connector_path, dockerfile_path), command, connector_config) async def do_teardown(container: dagger.Container, command: List[str]): diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml index 8a11d06427284..b4df3e3c5d65d 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector-acceptance-test" -version = "3.7.0" +version = "3.8.1" description = "Contains acceptance tests for connectors." authors = ["Airbyte "] license = "MIT"