From e2ba731e7acaec95eff345acbb18fb8970c727f3 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Thu, 15 Dec 2022 14:35:13 -0800 Subject: [PATCH 01/20] update SAT config --- .../acceptance-test-config.yml | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 1236f38994229..1f7e8510c6910 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -1,7 +1,28 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-postgres:dev -tests: +acceptance_tests: spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" + tests: + - spec_path: "src/test-integration/resources/expected_spec.json" + config_path: "src/test-integration/resources/dummy_config.json" + timeout_seconds: 60 + connection: + tests: + - config_path: "secrets/credentials.json" + status: "succeed" + timeout_seconds: 60 + discovery: + tests: + - config_path: "secrets/credentials.json" + timeout_seconds: 60 + basic_read: + tests: + - config_path: "secrets/credentials.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 120 + full_refresh: + tests: + - config_path: "secrets/credentials.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 180 \ No newline at end of file From 04014aa7c2fdbdb111c3103828f6b21aa1caa508 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Thu, 15 Dec 2022 14:48:35 -0800 Subject: [PATCH 02/20] update SAT config --- .../connectors/source-postgres/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 1f7e8510c6910..ee5e1026d2341 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-postgres:dev +test_strictness_level: high acceptance_tests: spec: tests: From b2a0ac3ddbb61eb1ef794d9a183a6ae715b5579c Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:28:50 -0800 Subject: [PATCH 03/20] update SAT config --- .../connectors/source-postgres/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index ee5e1026d2341..31c2db1fcd1c9 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -21,7 +21,7 @@ acceptance_tests: tests: - config_path: "secrets/credentials.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 120 + timeout_seconds: 180 full_refresh: tests: - config_path: "secrets/credentials.json" From b16785b154fbfc4f8cf4fe0c4ad1a2765261d609 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Thu, 15 Dec 2022 19:37:06 -0800 Subject: [PATCH 04/20] update SAT config --- .../acceptance-test-config.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 31c2db1fcd1c9..c41dbcade76e7 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -10,20 +10,13 @@ acceptance_tests: timeout_seconds: 60 connection: tests: - - config_path: "secrets/credentials.json" - status: "succeed" - timeout_seconds: 60 + bypass_reason: "some reason." discovery: tests: - - config_path: "secrets/credentials.json" - timeout_seconds: 60 + bypass_reason: "some reason." basic_read: - tests: - - config_path: "secrets/credentials.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 180 + bypass_reason: "some reason." full_refresh: - tests: - - config_path: "secrets/credentials.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 180 \ No newline at end of file + bypass_reason: "some reason." + incremental: + bypass_reason: "some reason." \ No newline at end of file From c048711c492dfd824108aca0507192791943a022 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 16 Dec 2022 09:53:43 +0100 Subject: [PATCH 05/20] fix acceptance-test-config structure --- .../connectors/source-postgres/acceptance-test-config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index c41dbcade76e7..3b095e2a45892 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -9,11 +9,9 @@ acceptance_tests: config_path: "src/test-integration/resources/dummy_config.json" timeout_seconds: 60 connection: - tests: - bypass_reason: "some reason." + bypass_reason: "some reason." discovery: - tests: - bypass_reason: "some reason." + bypass_reason: "some reason." basic_read: bypass_reason: "some reason." full_refresh: From ff79f9cf9a4d4c45cb71cd73891592fa22c9cac7 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 16 Dec 2022 10:30:10 +0100 Subject: [PATCH 06/20] enable more tests --- .../acceptance-test-config.yml | 20 ++++++++++++------- .../source-postgres/acceptance-test-docker.sh | 2 +- .../integration_tests/expected_records.txt | 3 +++ .../integration_tests/seed.sql | 2 ++ 4 files changed, 19 insertions(+), 8 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-postgres/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 3b095e2a45892..1182095213373 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -6,15 +6,21 @@ acceptance_tests: spec: tests: - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" - timeout_seconds: 60 + config_path: "secrets/config.json" connection: - bypass_reason: "some reason." + tests: + - config_path: "secrets/config.json" + status: "succeed" discovery: - bypass_reason: "some reason." + tests: + - config_path: "secrets/config.json" basic_read: - bypass_reason: "some reason." + tests: + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.txt" full_refresh: - bypass_reason: "some reason." + tests: + - config_path: "secrets/config.json" incremental: - bypass_reason: "some reason." \ No newline at end of file + bypass_reason: "The test database is not yet seeded with an incremental stream." \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh old mode 100644 new mode 100755 index ba0ab2874b989..4d7e4f45779b8 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev +#docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-postgres/integration_tests/expected_records.txt new file mode 100644 index 0000000000000..d886fbe3fe031 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/expected_records.txt @@ -0,0 +1,3 @@ +{"stream": "id_and_name", "data": {"id": 1, "name": "picard"}, "emitted_at": 999999} +{"stream": "id_and_name", "data": {"id": 2, "name": "crusher"}, "emitted_at": 999999} +{"stream": "id_and_name", "data": {"id": 3, "name": "vash"}, "emitted_at": 999999} diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql new file mode 100644 index 0000000000000..face983d6a8f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql @@ -0,0 +1,2 @@ +CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200)); +INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash'); From eca8af8357e0e4c8c02078e3f5d8a496395b002b Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 16 Dec 2022 10:35:29 +0100 Subject: [PATCH 07/20] clean --- .../connectors/source-postgres/acceptance-test-config.yml | 2 +- .../connectors/source-postgres/acceptance-test-docker.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 1182095213373..1915ee2574189 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -23,4 +23,4 @@ acceptance_tests: tests: - config_path: "secrets/config.json" incremental: - bypass_reason: "The test database is not yet seeded with an incremental stream." \ No newline at end of file + bypass_reason: "The test database is not yet seeded with an incremental stream." diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh index 4d7e4f45779b8..ba0ab2874b989 100755 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -#docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest From 776c78b1b86e7bdc3f4c23b545ecc5c03a036191 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 16 Dec 2022 10:57:41 +0100 Subject: [PATCH 08/20] set incremental test --- .../acceptance-test-config.yml | 8 ++++- .../integration_tests/abnormal_state.json | 13 ++++++++ .../incremental_configured_catalog.json | 32 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 1915ee2574189..6c9f8c9329516 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -23,4 +23,10 @@ acceptance_tests: tests: - config_path: "secrets/config.json" incremental: - bypass_reason: "The test database is not yet seeded with an incremental stream." + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + future_state: + missing_streams: + - name: id_and_name + bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..b64126b8f5fca --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json @@ -0,0 +1,13 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 4 + }, + "stream_descriptor": { + "name": "id_and_name" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json new file mode 100644 index 0000000000000..3ea2d1d8edd80 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json @@ -0,0 +1,32 @@ +{ + "streams": [ + { + "stream": { + "name": "id_and_name", + "json_schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "id": { + "type": "number", + "airbyte_type": "integer" + } + } + }, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "default_cursor_field": [], + "source_defined_primary_key": [], + "namespace": "public" + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"], + "user_defined_primary_key": ["id"] + } + ] +} \ No newline at end of file From f1e919e829dc3341e94904423690d77555391845 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 16 Dec 2022 11:07:45 +0100 Subject: [PATCH 09/20] clean --- .../integration_tests/incremental_configured_catalog.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json index 3ea2d1d8edd80..fe8fa5fc2372d 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json @@ -29,4 +29,4 @@ "user_defined_primary_key": ["id"] } ] -} \ No newline at end of file +} From 56a589f4b76a69abc17da2a38361181393f1237b Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 11:14:33 +0100 Subject: [PATCH 10/20] declare path to abnormal_state.json --- .../connectors/source-postgres/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 6c9f8c9329516..76b1cad7f10e1 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -27,6 +27,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/incremental_configured_catalog.json" future_state: + future_state_path: "integrations_tests/abnormal_state.json" missing_streams: - name: id_and_name bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" From 2ffad6327f1a8657bec385424fcdfd802b043f6a Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 12:14:38 +0100 Subject: [PATCH 11/20] add CDC config --- .../source-postgres/acceptance-test-config.yml | 18 +++++++++++++++++- .../source-postgres/integration_tests/seed.sql | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 76b1cad7f10e1..eeb75bff811fb 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -7,27 +7,43 @@ acceptance_tests: tests: - spec_path: "src/test-integration/resources/expected_spec.json" config_path: "secrets/config.json" + - spec_path: "src/test-integration/resources/expected_spec.json" + config_path: "secrets/config_cdc.json" connection: tests: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/config_cdc.json" + status: "succeed" discovery: tests: - config_path: "secrets/config.json" + - config_path: "secrets/config_cdc.json" basic_read: tests: - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.txt" + - config_path: "secrets/config_cdc.json" + expect_records: + path: "integration_tests/expected_records.txt" full_refresh: tests: - config_path: "secrets/config.json" + - config_path: "secrets/config_cdc.json" incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/incremental_configured_catalog.json" future_state: - future_state_path: "integrations_tests/abnormal_state.json" + future_state_path: "integration_tests/abnormal_state.json" + missing_streams: + - name: id_and_name + bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" + - config_path: "secrets/config_cdc.json" + configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" missing_streams: - name: id_and_name bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql index face983d6a8f5..feedf7663dbc7 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql @@ -1,2 +1,5 @@ +ALTER ROLE postgres WITH REPLICATION; CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200)); INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash'); +SELECT pg_create_logical_replication_slot('debezium_slot', 'pgoutput'); +CREATE PUBLICATION publication FOR ALL TABLES; \ No newline at end of file From 7a742858ea50ad48f82aece4775212a1a11e3395 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 15:42:57 +0100 Subject: [PATCH 12/20] declare bypass reason in future_state SAT --- .../source_acceptance_test/config.py | 1 + .../tests/test_incremental.py | 7 ++++--- .../unit_tests/test_incremental.py | 15 +++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py index 4c7cdf48cc038..ecfefb7fd59be 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py @@ -144,6 +144,7 @@ class FullRefreshConfig(BaseConfig): class FutureStateConfig(BaseConfig): future_state_path: Optional[str] = Field(description="Path to a state file with values in far future") missing_streams: List[EmptyStreamConfiguration] = Field(default=[], description="List of missings streams with valid bypass reasons.") + bypass_reason: Optional[str] class IncrementalConfig(BaseConfig): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py index 219bfc70e55f5..b345542840828 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py @@ -18,14 +18,15 @@ @pytest.fixture(name="future_state_configuration") def future_state_configuration_fixture(inputs, base_path, test_strictness_level) -> Tuple[Path, List[EmptyStreamConfiguration]]: """Fixture with connector's future state path (relative to base_path)""" - if inputs.future_state and inputs.future_state.future_state_path: + if inputs.future_state and inputs.future_state.bypass_reason is not None: + pytest.skip("`future_state` has a bypass reason, skipping.") + elif inputs.future_state and inputs.future_state.future_state_path: return Path(base_path) / inputs.future_state.future_state_path, inputs.future_state.missing_streams elif test_strictness_level is Config.TestStrictnessLevel.high: - pytest.fail("High test strictness level error: a future state configuration must be provided in high test strictness level.") + pytest.fail("High test strictness level error: a future state configuration must be provided in high test strictness level or a bypass reason should be filled.") else: pytest.skip("`future_state` not specified, skipping.") - @pytest.fixture(name="future_state") def future_state_fixture(future_state_configuration, test_strictness_level, configured_catalog) -> List[MutableMapping]: """""" diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py index 367a74c5ecb3a..f27c3c459b50c 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py @@ -695,32 +695,39 @@ def test_state_with_abnormally_large_values(mocker, read_output, expectation): [ pytest.param( Config.TestStrictnessLevel.high, - MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"])), + MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"], bypass_reason=None)), False, False, id="high test strictness level, future_state_path and missing streams are defined: run the test.", ), pytest.param( Config.TestStrictnessLevel.low, - MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"])), + MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"], bypass_reason=None)), False, False, id="low test strictness level, future_state_path and missing_streams are defined: run the test.", ), pytest.param( Config.TestStrictnessLevel.high, - MagicMock(future_state=MagicMock(future_state_path=None)), + MagicMock(future_state=MagicMock(future_state_path=None, bypass_reason=None)), True, False, id="high test strictness level, future_state_path and missing streams are defined: fail the test.", ), pytest.param( Config.TestStrictnessLevel.low, - MagicMock(future_state=MagicMock(future_state_path=None)), + MagicMock(future_state=MagicMock(future_state_path=None, bypass_reason=None)), False, True, id="low test strictness level, future_state_path not defined: skip the test.", ), + pytest.param( + Config.TestStrictnessLevel.high, + MagicMock(future_state=MagicMock(bypass_reason="valid bypass reason")), + False, + True, + id="high test strictness level, bypass_reason: skip test.", + ), ], ) def test_future_state_configuration_fixture(mocker, test_strictness_level, inputs, expect_fail, expect_skip): From 27f7cc7b4e80b2af413b14df33f1f7700a500717 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 15:45:57 +0100 Subject: [PATCH 13/20] bump SAT version --- airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md | 3 +++ airbyte-integrations/bases/source-acceptance-test/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index 193b110449f9d..bbde08b4dfa2f 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.2.24 +Enable bypass reason for future state test config.[#20549](https://github.com/airbytehq/airbyte/pull/20549) + ## 0.2.23 Skip backward compatibility tests on specifications if actual and previous specifications and discovered catalogs are identical.[#20435](https://github.com/airbytehq/airbyte/pull/20435) diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 24cc460a6f2e2..9d452200aec33 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.2.23 +LABEL io.airbyte.version=0.2.24 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] From 52121b1bea8577e7736be1b498164fcf411a0e6f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 15:53:37 +0100 Subject: [PATCH 14/20] set bypass reason postgres --- .../source-postgres/acceptance-test-config.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index eeb75bff811fb..4037858cb5330 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -36,14 +36,8 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/incremental_configured_catalog.json" future_state: - future_state_path: "integration_tests/abnormal_state.json" - missing_streams: - - name: id_and_name - bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" + bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" - config_path: "secrets/config_cdc.json" configured_catalog_path: "integration_tests/incremental_configured_catalog.json" future_state: - future_state_path: "integration_tests/abnormal_state.json" - missing_streams: - - name: id_and_name - bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" + bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" From 9442be5f2964e3ef8e2e3fa11792ee3526c231d6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 16:01:56 +0100 Subject: [PATCH 15/20] update SAT doc --- .../testing-connectors/source-acceptance-tests-reference.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md index afc7355fab7a1..fe480e9b2ccd0 100644 --- a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md @@ -236,6 +236,7 @@ This test verifies that sync produces no records when run with the STATE with ab | `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | | | `future_state_path` | string | None | Path to the state file with abnormally large cursor values | | | `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | | +| `bypass_reason` | string | None | Explain why this test is bypassed | | ## Strictness level From f6cebf50e573f18f40da39cd210b83b2d3211f5e Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 16:05:33 +0100 Subject: [PATCH 16/20] clean --- .../connectors/source-postgres/integration_tests/seed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql index feedf7663dbc7..07b41816ff9ce 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql @@ -2,4 +2,4 @@ ALTER ROLE postgres WITH REPLICATION; CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200)); INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash'); SELECT pg_create_logical_replication_slot('debezium_slot', 'pgoutput'); -CREATE PUBLICATION publication FOR ALL TABLES; \ No newline at end of file +CREATE PUBLICATION publication FOR ALL TABLES; From 77a05f22d3f5c8eb70b7f32aa3afae72e6c7b41d Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 19 Dec 2022 16:06:00 +0100 Subject: [PATCH 17/20] add integration-tests/README.md --- .../connectors/source-postgres/integration_tests/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 airbyte-integrations/connectors/source-postgres/integration_tests/README.md diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/README.md b/airbyte-integrations/connectors/source-postgres/integration_tests/README.md new file mode 100644 index 0000000000000..7c80177d8fd3e --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/README.md @@ -0,0 +1,5 @@ +This directory contains files used to run Source Acceptance Tests. +* `abnormal_state.json` describes a connector state with a non-existing cursor value. +* `expected_records.txt` lists all the records expected as the output of the basic read operation. +* `incremental_configured_catalog.json` is a configured catalog used as an input of the `incremental` test. +* `seed.sql` is the query we manually ran on a test postgres instance to seed it with test data and enable CDC. \ No newline at end of file From ba231a05e154bd0fcfb1623a1640b2e6523e4ad4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 4 Jan 2023 09:08:46 +0100 Subject: [PATCH 18/20] bump version --- airbyte-integrations/bases/source-acceptance-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 9d452200aec33..f740f5a6eb6ea 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.2.24 +LABEL io.airbyte.version=0.2.25 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] From 438b60864cae33921609387df465ee8ca6c148e6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 4 Jan 2023 09:22:18 +0100 Subject: [PATCH 19/20] format --- .../source_acceptance_test/tests/test_incremental.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py index b345542840828..5ca4e1932fb88 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py @@ -23,10 +23,13 @@ def future_state_configuration_fixture(inputs, base_path, test_strictness_level) elif inputs.future_state and inputs.future_state.future_state_path: return Path(base_path) / inputs.future_state.future_state_path, inputs.future_state.missing_streams elif test_strictness_level is Config.TestStrictnessLevel.high: - pytest.fail("High test strictness level error: a future state configuration must be provided in high test strictness level or a bypass reason should be filled.") + pytest.fail( + "High test strictness level error: a future state configuration must be provided in high test strictness level or a bypass reason should be filled." + ) else: pytest.skip("`future_state` not specified, skipping.") + @pytest.fixture(name="future_state") def future_state_fixture(future_state_configuration, test_strictness_level, configured_catalog) -> List[MutableMapping]: """""" From 2f659fb65f17f52e6eb625c432e6b68b07de7fe9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 4 Jan 2023 09:33:16 +0100 Subject: [PATCH 20/20] format SQL --- .../integration_tests/seed.sql | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql index 07b41816ff9ce..48910082f86fd 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql @@ -1,5 +1,36 @@ ALTER ROLE postgres WITH REPLICATION; -CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200)); -INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash'); -SELECT pg_create_logical_replication_slot('debezium_slot', 'pgoutput'); -CREATE PUBLICATION publication FOR ALL TABLES; + +CREATE + TABLE + id_and_name( + id INTEGER, + name VARCHAR(200) + ); + +INSERT + INTO + id_and_name( + id, + name + ) + VALUES( + 1, + 'picard' + ), + ( + 2, + 'crusher' + ), + ( + 3, + 'vash' + ); + +SELECT + pg_create_logical_replication_slot( + 'debezium_slot', + 'pgoutput' + ); + +CREATE + PUBLICATION publication FOR ALL TABLES;