Add validation test step

airbytehq · Jun 18, 2024 · 7ad39f1 · 7ad39f1
1 parent 77cbe6c
commit 7ad39f1
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 38 deletions.
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py
@@ -27,6 +27,7 @@ class CONNECTOR_TEST_STEP_ID(str, Enum):
     MIGRATE_POETRY_REGRESSION_TEST = "migrate_to_poetry.regression"
     CONNECTOR_LIVE_TESTS = "connector_live_tests"
     CONNECTOR_REGRESSION_TESTS = "connector_regression_tests"
+    CONNECTOR_VALIDATION_TESTS = "connector_validation_tests"
     REGRESSION_TEST = "common.regression_test"
     ADD_CHANGELOG_ENTRY = "bump_version.changelog"
     SET_CONNECTOR_VERSION = "bump_version.set"

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py
@@ -115,7 +115,7 @@ async def test(
         raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.")
     if not only_steps:
         skip_steps = list(skip_steps)
-        skip_steps += [CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS, CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS]
+        skip_steps += [CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS, CONNECTOR_TEST_STEP_ID.CONNECTOR_VALIDATION_TESTS, CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS]
     if ctx.obj["is_ci"]:
         fail_if_missing_docker_hub_creds(ctx)
 

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py
@@ -443,14 +443,13 @@ async def _run(self, current_acceptance_tests_result: StepResult) -> StepResult:
             )
 
 
-class LiveTests(Step):
+class AbstractLiveTests(Step):
     """A step to run live tests for a connector."""
 
     context: ConnectorContext
-    title = "Live tests (regression + validation)"
     skipped_exit_code = 5
     accept_extra_params = True
-    tests_artifacts_dir = Path("/tmp/regression_tests_artifacts")
+    tests_artifacts_dir = Path("/tmp/live_tests_artifacts")
     working_directory = "/app"
     github_user = "octavia-squidington-iii"
     platform_repo_url = "airbytehq/airbyte-platform-internal"
@@ -468,33 +467,20 @@ def default_params(self) -> STEP_PARAMS:
             "--durations": ["3"],  # Show the 3 slowest tests in the report
         }
 
-    def _test_command(self) -> str:
-        selected_streams = ["--stream", self.selected_streams] if self.selected_streams else []
-        return " ".join(
-            [
-                "poetry",
-                "run",
-                "pytest",
-                "src/live_tests",
-                "--connector-image",
-                self.connector_image,
-                "--connection-id",
-                self.connection_id or "",
-                "--control-version",
-                self.control_version or "",
-                "--target-version",
-                self.target_version or "",
-                "--pr-url",
-                self.pr_url or "",
-                "--run-id",
-                self.run_id or "",
-                "--should-read-with-state",
-                str(self.should_read_with_state),
-                "--test-evaluation-mode",
-                str(self.test_evaluation_mode),
-            ]
-            + selected_streams
-        )
+    @property
+    @abstractmethod
+    def title(self):
+        """
+        The title of the tests, for report display purposes.
+        """
+        ...
+
+    @abstractmethod
+    def _test_command(self):
+        """
+        The command used to run the tests
+        """
+        ...
 
     def _run_command_with_proxy(self, command: str) -> List[str]:
         """
@@ -526,20 +512,20 @@ def _run_command_with_proxy(self, command: str) -> List[str]:
         return ["bash", "-c", f"'{run_pytest_with_proxy}'"]
 
     def __init__(self, context: ConnectorContext) -> None:
-        """Create a step to run regression tests for a connector.
+        """Create a step to run live tests for a connector.
 
         Args:
             context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory.
         """
         super().__init__(context)
         self.connector_image = context.docker_image.split(":")[0]
-        options = self.context.run_step_options.step_params.get(CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS, {})
+        options = self.context.run_step_options.step_params.get(CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS, {})
 
         self.connection_id = self.context.run_step_options.get_item_or_default(options, "connection-id", None)
         self.pr_url = self.context.run_step_options.get_item_or_default(options, "pr-url", None)
 
         if not self.connection_id and self.pr_url:
-            raise ValueError("`connection-id` and `pr-url` are required to run regression tests.")
+            raise ValueError("`connection-id` and `pr-url` are required to run live tests.")
 
         self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest")
         self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev")
@@ -676,10 +662,42 @@ async def _build_test_container(self, target_container_id: str) -> Container:
         return container
 
 
-class RegressionTests(LiveTests):
-    """A step to run live tests for a connector."""
+class LiveTests(AbstractLiveTests):
+
+    title = "Live tests (regression + validation)"
+
+    def _test_command(self) -> str:
+        selected_streams = ["--stream", self.selected_streams] if self.selected_streams else []
+        return " ".join(
+            [
+                "poetry",
+                "run",
+                "pytest",
+                "src/live_tests",
+                "--connector-image",
+                self.connector_image,
+                "--connection-id",
+                self.connection_id or "",
+                "--control-version",
+                self.control_version or "",
+                "--target-version",
+                self.target_version or "",
+                "--pr-url",
+                self.pr_url or "",
+                "--run-id",
+                self.run_id or "",
+                "--should-read-with-state",
+                str(self.should_read_with_state),
+                "--test-evaluation-mode",
+                str(self.test_evaluation_mode),
+            ]
+            + selected_streams
+        )
+
+
+class RegressionTests(AbstractLiveTests):
+    """A step to run regression tests for a connector."""
 
-    context: ConnectorContext
     title = "Regression tests"
 
     def _test_command(self) -> str:
@@ -710,3 +728,35 @@ def _test_command(self) -> str:
             + selected_streams
         )
 
+
+class ValidationTests(AbstractLiveTests):
+    """A step to run validation tests for a connector."""
+
+    title = "Validation tests"
+
+    def _test_command(self) -> str:
+        selected_streams = ["--stream", self.selected_streams] if self.selected_streams else []
+        return " ".join(
+            [
+                "poetry",
+                "run",
+                "pytest",
+                "src/live_tests/validation_tests",
+                "--connector-image",
+                self.connector_image,
+                "--connection-id",
+                self.connection_id or "",
+                "--target-version",
+                self.target_version or "",
+                "--pr-url",
+                self.pr_url or "",
+                "--run-id",
+                self.run_id or "",
+                "--should-read-with-state",
+                str(self.should_read_with_state),
+                "--test-evaluation-mode",
+                str(self.test_evaluation_mode),
+            ]
+            + selected_streams
+        )
+
diff --git a/...e-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/...e-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py
@@ -15,7 +15,7 @@
 from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages
 from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID
 from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext
-from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, IncrementalAcceptanceTests, RegressionTests
+from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, IncrementalAcceptanceTests, LiveTests, RegressionTests, ValidationTests
 from pipelines.consts import LOCAL_BUILD_PLATFORM
 from pipelines.dagger.actions import secrets
 from pipelines.dagger.actions.python.poetry import with_poetry
@@ -292,6 +292,12 @@ def get_test_steps(context: ConnectorTestContext) -> STEP_TREE:
                 args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]},
                 depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
             ),
+            StepToRun(
+                id=CONNECTOR_TEST_STEP_ID.CONNECTOR_VALIDATION_TESTS,
+                step=ValidationTests(context),
+                args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]},
+                depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
+            ),
             StepToRun(
                 id=CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
                 step=LiveTests(context),