Single live tests CI command

.github/workflows/validation_tests.yml → .github/workflows/live_tests.yml

@@ -102,4 +102,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} --connector_live_tests.test-evaluation-mode=diagnostic
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=all --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} --connector_live_tests.test-evaluation-mode=diagnostic

.github/workflows/regression_tests.yml

@@ -73,8 +73,8 @@ jobs:
           if [ -z "${{ github.event.inputs.streams }}" ]; then
             echo "STREAM_PARAMS=" >> $GITHUB_ENV
           else
-            STREAMS=$(echo "${{ github.event.inputs.streams }}" | sed 's/,/ --connector_regression_tests.selected-streams=/g')
-            echo "STREAM_PARAMS=--connector_regression_tests.selected-streams=$STREAMS" >> $GITHUB_ENV
+            STREAMS=$(echo "${{ github.event.inputs.streams }}" | sed 's/,/ --connector_live_tests.selected-streams=/g')
+            echo "STREAM_PARAMS=--connector_live_tests.selected-streams=$STREAMS" >> $GITHUB_ENV
           fi
 
       - name: Setup Local CDK Flag
@@ -102,4 +102,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_regression_tests --connector_regression_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_regression_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }}
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }}

airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py

@@ -16,7 +16,13 @@
     SyncMode,
     Type,
 )
-from live_tests.commons.json_schema_helper import ComparableType, JsonSchemaHelper, conforms_to_schema, get_expected_schema_structure, get_object_structure
+from live_tests.commons.json_schema_helper import (
+    ComparableType,
+    JsonSchemaHelper,
+    conforms_to_schema,
+    get_expected_schema_structure,
+    get_object_structure,
+)
 from pydantic import BaseModel
 
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py

@@ -26,8 +26,6 @@ class CONNECTOR_TEST_STEP_ID(str, Enum):
     MIGRATE_POETRY_DELETE_SETUP_PY = "migrate_to_poetry.delete_setup_py"
     MIGRATE_POETRY_REGRESSION_TEST = "migrate_to_poetry.regression"
     CONNECTOR_LIVE_TESTS = "connector_live_tests"
-    CONNECTOR_REGRESSION_TESTS = "connector_regression_tests"
-    CONNECTOR_VALIDATION_TESTS = "connector_validation_tests"
     REGRESSION_TEST = "common.regression_test"
     ADD_CHANGELOG_ENTRY = "bump_version.changelog"
     SET_CONNECTOR_VERSION = "bump_version.set"

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

@@ -11,7 +11,7 @@
 from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
 from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext
 from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline
-from pipelines.airbyte_ci.connectors.test.steps.common import RegressionTests
+from pipelines.airbyte_ci.connectors.test.steps.common import LiveTests
 from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 from pipelines.consts import LOCAL_BUILD_PLATFORM, MAIN_CONNECTOR_TESTING_SECRET_STORE_ALIAS, ContextState
@@ -25,8 +25,6 @@
 GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
 GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
 TESTS_SKIPPED_BY_DEFAULT = [
-    CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS,
-    CONNECTOR_TEST_STEP_ID.CONNECTOR_VALIDATION_TESTS,
     CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
 ]
 
@@ -186,9 +184,9 @@ async def test(
         return False
 
     finally:
-        if RegressionTests.regression_tests_artifacts_dir.exists():
-            shutil.rmtree(RegressionTests.regression_tests_artifacts_dir)
-            main_logger.info(f"  Test artifacts cleaned up from {RegressionTests.regression_tests_artifacts_dir}")
+        if LiveTests.local_tests_artifacts_dir.exists():
+            shutil.rmtree(LiveTests.local_tests_artifacts_dir)
+            main_logger.info(f"  Test artifacts cleaned up from {LiveTests.local_tests_artifacts_dir}")
 
     @ctx.call_on_close
     def send_commit_status_check() -> None:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

@@ -9,6 +9,7 @@
 import os
 import time
 from abc import ABC, abstractmethod
+from enum import Enum
 from functools import cached_property
 from pathlib import Path
 from textwrap import dedent
@@ -443,16 +444,27 @@ async def _run(self, current_acceptance_tests_result: StepResult) -> StepResult:
             )
 
 
-class AbstractLiveTests(Step):
+class LiveTestSuite(Enum):
+    ALL = "all"
+    REGRESSION = "regression"
+    VALIDATION = "validation"
+
+
+class LiveTests(Step):
     """A step to run live tests for a connector."""
 
     context: ConnectorContext
     skipped_exit_code = 5
     accept_extra_params = True
-    in_container_tests_artifacts_dir = Path("/tmp/live_tests_artifacts")
+    local_tests_artifacts_dir = Path("/tmp/live_tests_artifacts")
     working_directory = "/app"
     github_user = "octavia-squidington-iii"
     platform_repo_url = "airbytehq/airbyte-platform-internal"
+    test_suite_to_dir = {
+        LiveTestSuite.ALL: "src/live_tests",
+        LiveTestSuite.REGRESSION: "src/live_tests/regression_tests",
+        LiveTestSuite.VALIDATION: "src/live_tests/validation_tests",
+    }
 
     @property
     def default_params(self) -> STEP_PARAMS:
@@ -468,12 +480,8 @@ def default_params(self) -> STEP_PARAMS:
         }
 
     @property
-    @abstractmethod
-    def test_dir(self) -> str:
-        """
-        The directory of tests to run.
-        """
-        ...
+    def title(self) -> str:
+        return f"Connector {self.test_suite.title()} Tests"
 
     def _test_command(self) -> List[str]:
         """
@@ -552,9 +560,11 @@ def __init__(self, context: ConnectorContext) -> None:
         if not self.connection_id and self.pr_url:
             raise ValueError("`connection-id` and `pr-url` are required to run live tests.")
 
+        self.test_suite = self.context.run_step_options.get_item_or_default(options, "test-suite", LiveTestSuite.ALL.value)
+        self.test_dir = self.test_suite_to_dir[LiveTestSuite(self.test_suite)]
         self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest")
         self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev")
-        self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", True)
+        self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", "1")
         self.selected_streams = self.context.run_step_options.get_item_or_default(options, "selected-streams", None)
         self.test_evaluation_mode = self.context.run_step_options.get_item_or_default(options, "test-evaluation-mode", "strict")
         self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time()))
@@ -570,7 +580,7 @@ async def _run(self, connector_under_test_container: Container) -> StepResult:
         """
         container = await self._build_test_container(await connector_under_test_container.id())
         container = container.with_(hacks.never_fail_exec(self._run_command_with_proxy(" ".join(self._test_command()))))
-        tests_artifacts_dir = str(self.in_container_tests_artifacts_dir)
+        tests_artifacts_dir = str(self.local_tests_artifacts_dir)
         path_to_report = f"{tests_artifacts_dir}/session_{self.run_id}/report.html"
 
         exit_code, stdout, stderr = await get_exec_result(container)
@@ -685,23 +695,3 @@ async def _build_test_container(self, target_container_id: str) -> Container:
 
         container = container.with_exec(["poetry", "lock", "--no-update"]).with_exec(["poetry", "install"])
         return container
-
-
-class LiveTests(AbstractLiveTests):
-
-    title = "Live tests (regression + validation)"
-    test_dir = "src/live_tests"
-
-
-class RegressionTests(AbstractLiveTests):
-    """A step to run regression tests for a connector."""
-
-    title = "Regression tests"
-    test_dir = "src/live_tests/regression_tests"
-
-
-class ValidationTests(AbstractLiveTests):
-    """A step to run validation tests for a connector."""
-
-    title = "Validation tests"
-    test_dir = "src/live_tests/validation_tests"

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py

@@ -15,7 +15,7 @@
 from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages
 from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID
 from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext
-from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, IncrementalAcceptanceTests, LiveTests, RegressionTests, ValidationTests
+from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, IncrementalAcceptanceTests, LiveTests
 from pipelines.consts import LOCAL_BUILD_PLATFORM
 from pipelines.dagger.actions import secrets
 from pipelines.dagger.actions.python.poetry import with_poetry
@@ -286,18 +286,6 @@ def get_test_steps(context: ConnectorTestContext) -> STEP_TREE:
                 args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]},
                 depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
             ),
-            StepToRun(
-                id=CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS,
-                step=RegressionTests(context),
-                args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]},
-                depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
-            ),
-            StepToRun(
-                id=CONNECTOR_TEST_STEP_ID.CONNECTOR_VALIDATION_TESTS,
-                step=ValidationTests(context),
-                args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]},
-                depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
-            ),
             StepToRun(
                 id=CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
                 step=LiveTests(context),