CATs: add connector code to setup container (#40613)

airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.8.1
+
+Provide the ability for users to perform setup/teardown by building and running a container before each test.
+
 ## 3.8.0
 
 Add `TestDiscovery.test_primary_keys_data_type`, which validates that primary keys are not of type `array` or `object` in discovered catalog.

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py

@@ -212,6 +212,7 @@ async def setup_and_teardown(
         logging.info("Running setup")
         setup_teardown_container = await setup_and_teardown_runner.do_setup(
             dagger_client,
+            base_path,
             base_path / setup_teardown_dockerfile_config.setup_teardown_dockerfile_path,
             setup_teardown_dockerfile_config.setup_command,
             connector_config,

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py

@@ -4,6 +4,7 @@
 
 import json
 import os
+from glob import glob
 from pathlib import Path
 from typing import List
 
@@ -17,16 +18,33 @@
 async def _build_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container:
     workspace = (
         dagger_client.container()
-        .with_directory("/tmp", dagger_client.host().directory(os.path.dirname(dockerfile_path)))
-        .with_workdir("/tmp")
-        .with_file("/tmp/Dockerfile.cat_setup_teardown", dagger_client.host().file(str(dockerfile_path.expanduser())))
-        .directory("/tmp")
+        .with_mounted_directory("/tmp/setup_teardown_context", dagger_client.host().directory(os.path.dirname(dockerfile_path)))
+        .directory("/tmp/setup_teardown_context")
     )
     return await dagger_client.container().build(context=workspace, dockerfile="Dockerfile.cat_setup_teardown")
 
 
-async def _build_setup_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container:
-    return await _build_container(dagger_client, dockerfile_path)
+async def _build_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path) -> dagger.Container:
+    container = await _build_container(dagger_client, dockerfile_path)
+    return container.with_mounted_directory(
+        "/connector", dagger_client.host().directory(str(connector_path), exclude=get_default_excluded_files())
+    )
+
+
+def get_default_excluded_files() -> List[str]:
+    return (
+        [".git"]
+        + glob("**/build", recursive=True)
+        + glob("**/.venv", recursive=True)
+        + glob("**/__pycache__", recursive=True)
+        + glob("**/*.egg-info", recursive=True)
+        + glob("**/.vscode", recursive=True)
+        + glob("**/.pytest_cache", recursive=True)
+        + glob("**/.eggs", recursive=True)
+        + glob("**/.mypy_cache", recursive=True)
+        + glob("**/.DS_Store", recursive=True)
+        + glob("**/.gradle", recursive=True)
+    )
 
 
 async def _run_with_config(container: dagger.Container, command: List[str], config: SecretDict) -> dagger.Container:
@@ -38,8 +56,10 @@ async def _run(container: dagger.Container, command: List[str]) -> dagger.Contai
     return await container.with_exec(command, skip_entrypoint=True)
 
 
-async def do_setup(dagger_client: dagger.Client, dockerfile_path: Path, command: List[str], connector_config: SecretDict):
-    return await _run_with_config(await _build_setup_container(dagger_client, dockerfile_path), command, connector_config)
+async def do_setup(
+    dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path, command: List[str], connector_config: SecretDict
+):
+    return await _run_with_config(await _build_client_container(dagger_client, connector_path, dockerfile_path), command, connector_config)
 
 
 async def do_teardown(container: dagger.Container, command: List[str]):

airbyte-integrations/bases/connector-acceptance-test/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector-acceptance-test"
-version = "3.7.0"
+version = "3.8.1"
 description = "Contains acceptance tests for connectors."
 authors = ["Airbyte <contact@airbyte.io>"]
 license = "MIT"