From 759d743d7f8f874f19b62110b881e7b5f8a25afe Mon Sep 17 00:00:00 2001 From: selfeer Date: Tue, 23 Jul 2024 21:57:12 +0400 Subject: [PATCH 01/44] add some new tests for replication --- .../tests/integration/tests/autocreate.py | 41 ++++++++++++++++++- .../tests/integration/tests/steps/sql.py | 3 +- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index 1f4b83898..c3003a924 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -4,7 +4,7 @@ @TestOutline -def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_table): +def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_table, auto_create_replicated=False): """Check auto-creation of replicated MySQL table which contains all supported data types. """ @@ -13,7 +13,7 @@ def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_ta mysql = self.context.cluster.node("mysql-master") init_sink_connector( - auto_create_tables=clickhouse_table[0], topics=f"SERVER5432.test.{table_name}" + auto_create_tables=clickhouse_table[0], topics=f"SERVER5432.test.{table_name}", auto_create_replicated_tables=auto_create_replicated ) with Given( @@ -43,6 +43,7 @@ def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_ta clickhouse_table=clickhouse_table, statement="count(*)", with_final=True, + replicated=auto_create_replicated ) @@ -64,6 +65,24 @@ def create_all_data_types_null_table( clickhouse_table=clickhouse_table, ) +@TestFeature +def create_all_data_types_null_table_replicated( + self, + mysql_columns=all_mysql_datatypes, + clickhouse_columns=all_ch_datatypes, +): + """Check all availabe methods and tables creation of replicated MySQL to Ch table that + contains all supported "NULL" data types. + """ + + for clickhouse_table in available_clickhouse_tables: + with Example({clickhouse_table}, flags=TE): + create_all_data_types( + mysql_columns=mysql_columns, + clickhouse_columns=clickhouse_columns, + clickhouse_table=clickhouse_table, + auto_create_replicated=True, + ) @TestFeature def create_all_data_types_not_null_table_manual( @@ -82,6 +101,24 @@ def create_all_data_types_not_null_table_manual( clickhouse_table=clickhouse_table, ) +@TestFeature +def create_all_data_types_not_null_table_manual_replicated( + self, + mysql_columns=all_nullable_mysql_datatypes, + clickhouse_columns=all_nullable_ch_datatypes, +): + """Check all availabe methods and tables creation of replicated MySQL to CH table + which contains all supported "NOT NULL" data types. + """ + for clickhouse_table in available_clickhouse_tables: + with Example({clickhouse_table}, flags=TE): + create_all_data_types( + mysql_columns=mysql_columns, + clickhouse_columns=clickhouse_columns, + clickhouse_table=clickhouse_table, + auto_create_replicated=True, + ) + @TestModule @Requirements( diff --git a/sink-connector/tests/integration/tests/steps/sql.py b/sink-connector/tests/integration/tests/steps/sql.py index 49e94dedd..f93e903ad 100644 --- a/sink-connector/tests/integration/tests/steps/sql.py +++ b/sink-connector/tests/integration/tests/steps/sql.py @@ -297,6 +297,7 @@ def complex_check_creation_and_select( with_final=False, with_optimize=False, order_by=None, + replicated=False ): """ Check for table creation on all clickhouse nodes where it is expected and select data consistency with MySql @@ -315,7 +316,7 @@ def complex_check_creation_and_select( clickhouse3 = self.context.cluster.node("clickhouse3") mysql = self.context.cluster.node("mysql-master") - if clickhouse_table[1].startswith("Replicated"): + if replicated: with Then("I check table creation on few nodes"): retry(clickhouse.query, timeout=30, delay=3)( "SHOW TABLES FROM test", message=f"{table_name}" From 584c2927f9e636fe04fe84c53493f311ebdf8bbf Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 15:50:04 +0400 Subject: [PATCH 02/44] update requirements --- .../tests/integration/tests/steps/sql.py | 36 +++++++++---------- .../integration/tests/steps/steps_global.py | 4 +-- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/sink-connector/tests/integration/tests/steps/sql.py b/sink-connector/tests/integration/tests/steps/sql.py index f93e903ad..c6a501f7d 100644 --- a/sink-connector/tests/integration/tests/steps/sql.py +++ b/sink-connector/tests/integration/tests/steps/sql.py @@ -225,7 +225,7 @@ def select( with_optimize=False, sign_column="_sign", timeout=100, - order_by=None + order_by=None, ): """SELECT with an option to either with FINAL or loop SELECT + OPTIMIZE TABLE default simple 'SELECT' :param insert: expected insert data if None compare with MySQL table @@ -293,11 +293,14 @@ def complex_check_creation_and_select( clickhouse_table, statement, timeout=50, + message=1, + clickhouse_node=None, + database_name=None, manual_output=None, with_final=False, with_optimize=False, order_by=None, - replicated=False + replicated=False, ): """ Check for table creation on all clickhouse nodes where it is expected and select data consistency with MySql @@ -310,29 +313,22 @@ def complex_check_creation_and_select( :param with_optimize: :return: """ - clickhouse = self.context.cluster.node("clickhouse") - clickhouse1 = self.context.cluster.node("clickhouse1") - clickhouse2 = self.context.cluster.node("clickhouse2") - clickhouse3 = self.context.cluster.node("clickhouse3") - mysql = self.context.cluster.node("mysql-master") + if clickhouse_node is None: + clickhouse_node = self.context.cluster.node("clickhouse") + + if database_name is None: + database_name = "test" if replicated: with Then("I check table creation on few nodes"): - retry(clickhouse.query, timeout=30, delay=3)( - "SHOW TABLES FROM test", message=f"{table_name}" - ) - retry(clickhouse1.query, timeout=30, delay=3)( - "SHOW TABLES FROM test", message=f"{table_name}" - ) - retry(clickhouse2.query, timeout=30, delay=3)( - "SHOW TABLES FROM test", message=f"{table_name}" - ) - retry(clickhouse3.query, timeout=30, delay=3)( - "SHOW TABLES FROM test", message=f"{table_name}" - ) + for node in self.context.cluster.nodes["clickhouse"]: + retry(self.context.cluster.node(node).query, timeout=timeout, delay=3)( + f"EXISTS {database_name}.{table_name}", message=f"{message}" + ) + else: with Then("I check table creation"): - retry(clickhouse.query, timeout=30, delay=3)( + retry(clickhouse_node.query, timeout=30, delay=3)( "SHOW TABLES FROM test", message=f"{table_name}" ) diff --git a/sink-connector/tests/integration/tests/steps/steps_global.py b/sink-connector/tests/integration/tests/steps/steps_global.py index faebf2d92..228737428 100644 --- a/sink-connector/tests/integration/tests/steps/steps_global.py +++ b/sink-connector/tests/integration/tests/steps/steps_global.py @@ -15,6 +15,4 @@ def create_database(self, name="test", node=None): yield finally: with Finally(f"I delete {name} database if exists"): - node.query( - f"DROP DATABASE IF EXISTS {name} ON CLUSTER replicated_cluster;" - ) + node.query(f"DROP DATABASE IF EXISTS {name} ON CLUSTER replicated_cluster;") From 74fab5884f4cae3eab68e632fe567f3f61478ed2 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 17:46:54 +0400 Subject: [PATCH 03/44] update regression and CI/CD for kafka --- .../testflows-sink-connector-kafka.yml | 15 ++- .../tests/integration/env/docker-compose.yml | 96 +++---------------- .../tests/integration/regression.py | 4 +- .../tests/steps/service_settings_steps.py | 4 +- 4 files changed, 32 insertions(+), 87 deletions(-) diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index ac2963adf..66bd21c85 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -1,4 +1,5 @@ name: Kafka - TestFlows Tests +run-name: ${{ inputs.custom_run_name || 'Kafka - TestFlows Tests' }} on: workflow_call: @@ -22,7 +23,17 @@ on: description: "Kafka connector docker image" required: true type: string - + package: + description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'" + type: string + default: docker://clickhouse/clickhouse-server:23.3 + extra_args: + description: "Specific Suite To Run (Default * to run everything)." + required: false + type: string + custom_run_name: + description: 'Custom run name (optional)' + required: false env: SINK_CONNECTOR_IMAGE: ${{ inputs.SINK_CONNECTOR_IMAGE }} @@ -75,7 +86,7 @@ jobs: - name: Run testflows tests working-directory: sink-connector/tests/integration - run: python3 -u regression.py --only "/mysql to clickhouse replication/*" --clickhouse-binary-path=docker://clickhouse/clickhouse-server:22.8 --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + run: python3 -u regression.py --only "/regression/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log - name: Create tfs results report if: always() diff --git a/sink-connector/tests/integration/env/docker-compose.yml b/sink-connector/tests/integration/env/docker-compose.yml index 7c6733b67..469065bfc 100644 --- a/sink-connector/tests/integration/env/docker-compose.yml +++ b/sink-connector/tests/integration/env/docker-compose.yml @@ -3,26 +3,9 @@ version: "2.3" services: mysql-master: - container_name: mysql-master - image: docker.io/bitnami/mysql:8.0.36 - restart: "no" - expose: - - "3306" - environment: - - MYSQL_ROOT_PASSWORD=root - - MYSQL_DATABASE=test - - MYSQL_REPLICATION_MODE=master - - MYSQL_REPLICATION_USER=repl_user - - ALLOW_EMPTY_PASSWORD=yes - volumes: - - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf - - ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql - - "${CLICKHOUSE_TESTS_DIR}/_instances/share_folder:/tmp/share_folder" - healthcheck: - test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ] - interval: 15s - timeout: 5s - retries: 6 + extends: + file: mysql-master-service.yml + service: mysql-master schemaregistry: @@ -36,70 +19,25 @@ services: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_LISTENERS=http://schemaregistry:8081 - SCHEMA_REGISTRY_DEBUG=true - depends_on: - kafka debezium: - container_name: debezium - hostname: debezium -# image: debezium/connect:1.9.5.Final - build: - context: ../../../docker/debezium_jmx - args: - DEBEZIUM_VERSION: 2.1.0.Alpha1 - restart: "no" - expose: - - "8083" - - "1976" - environment: - - BOOTSTRAP_SERVERS=kafka:9092 - - GROUP_ID=1 - - CONFIG_STORAGE_TOPIC=config-storage-topic-debezium - - OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium - - STATUS_STORAGE_TOPIC=status-storage-topic-debezium - - LOG_LEVEL=INFO - - KEY_CONVERTER=io.confluent.connect.avro.AvroConverter - - VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter + extends: + file: debezium-service.yml + service: debezium depends_on: - kafka kafka: - container_name: kafka - hostname: kafka - image: vectorized/redpanda - restart: "no" - expose: - - "19092" - command: - - redpanda - - start - - --overprovisioned - - --kafka-addr - - DOCKER_NETWORK://0.0.0.0:9092,LOCALHOST_NETWORK://0.0.0.0:19092 - - --advertise-kafka-addr - - DOCKER_NETWORK://kafka:9092,LOCALHOST_NETWORK://127.0.0.1:19092 - - sink: - container_name: sink - hostname: sink - image: ${SINK_CONNECTOR_IMAGE} - restart: "no" - expose: - - "8083" - - "5005" - - "39999" - environment: - - BOOTSTRAP_SERVERS=kafka:9092 - - GROUP_ID=2 - - CONFIG_STORAGE_TOPIC=config-storage-topic-sink - - OFFSET_STORAGE_TOPIC=offset-storage-topic-sink - - STATUS_STORAGE_TOPIC=status-storage-topic-sink - - LOG_LEVEL=INFO - - JAVA_DEBUG_PORT=*:5005 - - DEFAULT_JAVA_DEBUG_PORT=*:5005 - - KAFKA_DEBUG=true - - JMX_PORT=39999 + extends: + file: kafka-service.yml + service: kafka + + clickhouse-sink-connector-kafka: + extends: + file: clickhouse-sink-connector-kafka-service.yml + service: clickhouse-sink-connector-kafka depends_on: - kafka @@ -114,10 +52,6 @@ services: file: clickhouse-service.yml service: clickhouse hostname: clickhouse -# environment: -# - CLICKHOUSE_USER=1000 -# - CLICKHOUSE_PASSWORD=1000 -# - CLICKHOUSE_DB=test ulimits: nofile: soft: 262144 @@ -186,7 +120,7 @@ services: condition: service_healthy # dummy service which does nothing, but allows to postpone - # 'docker-compose up -d' till all dependecies will go healthy + # 'docker-compose up -d' till all dependencies will go healthy all_services_ready: image: hello-world depends_on: diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index 3d01cb4bd..e806e39e3 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -48,7 +48,7 @@ @ArgumentParser(argparser) @XFails(xfails) @XFlags(xflags) -@Name("mysql to clickhouse replication") +@Name("regression") @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication("1.0"), RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Consistency_Select("1.0"), @@ -74,7 +74,7 @@ def regression( "clickhouse": ("clickhouse", "clickhouse1", "clickhouse2", "clickhouse3"), "bash-tools": ("bash-tools",), "schemaregistry": ("schemaregistry",), - "sink": ("sink",), + "clickhouse-sink-connector-kafka": ("clickhouse-sink-connector-kafka",), "zookeeper": ("zookeeper",), } diff --git a/sink-connector/tests/integration/tests/steps/service_settings_steps.py b/sink-connector/tests/integration/tests/steps/service_settings_steps.py index 5230ac8d6..bda3f872e 100644 --- a/sink-connector/tests/integration/tests/steps/service_settings_steps.py +++ b/sink-connector/tests/integration/tests/steps/service_settings_steps.py @@ -29,7 +29,7 @@ def init_sink_connector( # "topics": "SERVER5432.test.users", sink_settings_transfer_command_confluent = ( - """cat </dev/null | jq ." ) From 309338800ec585148546a510994f86ff73bb1eb8 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 17:48:49 +0400 Subject: [PATCH 04/44] move docker compose services in to different files --- ...lickhouse-sink-connector-kafka-service.yml | 22 +++++++++++++++++ .../integration/env/debezium-service.yml | 23 ++++++++++++++++++ .../tests/integration/env/kafka-service.yml | 18 ++++++++++++++ .../integration/env/mysql-master-service.yml | 24 +++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 sink-connector/tests/integration/env/clickhouse-sink-connector-kafka-service.yml create mode 100644 sink-connector/tests/integration/env/debezium-service.yml create mode 100644 sink-connector/tests/integration/env/kafka-service.yml create mode 100644 sink-connector/tests/integration/env/mysql-master-service.yml diff --git a/sink-connector/tests/integration/env/clickhouse-sink-connector-kafka-service.yml b/sink-connector/tests/integration/env/clickhouse-sink-connector-kafka-service.yml new file mode 100644 index 000000000..3a6379621 --- /dev/null +++ b/sink-connector/tests/integration/env/clickhouse-sink-connector-kafka-service.yml @@ -0,0 +1,22 @@ +version: "2.3" + +services: + clickhouse-sink-connector-kafka: + hostname: clickhouse-sink-connector-kafka + image: ${SINK_CONNECTOR_IMAGE} + restart: "no" + expose: + - "8083" + - "5005" + - "39999" + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=2 + - CONFIG_STORAGE_TOPIC=config-storage-topic-sink + - OFFSET_STORAGE_TOPIC=offset-storage-topic-sink + - STATUS_STORAGE_TOPIC=status-storage-topic-sink + - LOG_LEVEL=INFO + - JAVA_DEBUG_PORT=*:5005 + - DEFAULT_JAVA_DEBUG_PORT=*:5005 + - KAFKA_DEBUG=true + - JMX_PORT=39999 \ No newline at end of file diff --git a/sink-connector/tests/integration/env/debezium-service.yml b/sink-connector/tests/integration/env/debezium-service.yml new file mode 100644 index 000000000..0b74a6e2e --- /dev/null +++ b/sink-connector/tests/integration/env/debezium-service.yml @@ -0,0 +1,23 @@ +version: "2.3" + +services: + debezium: + container_name: debezium + hostname: debezium + build: + context: ../../../docker/debezium_jmx + args: + DEBEZIUM_VERSION: 2.1.0.Alpha1 + restart: "no" + expose: + - "8083" + - "1976" + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=1 + - CONFIG_STORAGE_TOPIC=config-storage-topic-debezium + - OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium + - STATUS_STORAGE_TOPIC=status-storage-topic-debezium + - LOG_LEVEL=INFO + - KEY_CONVERTER=io.confluent.connect.avro.AvroConverter + - VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter \ No newline at end of file diff --git a/sink-connector/tests/integration/env/kafka-service.yml b/sink-connector/tests/integration/env/kafka-service.yml new file mode 100644 index 000000000..f8b3d24d4 --- /dev/null +++ b/sink-connector/tests/integration/env/kafka-service.yml @@ -0,0 +1,18 @@ +version: "2.3" + +services: + kafka: + container_name: kafka + hostname: kafka + image: vectorized/redpanda + restart: "no" + expose: + - "19092" + command: + - redpanda + - start + - --overprovisioned + - --kafka-addr + - DOCKER_NETWORK://0.0.0.0:9092,LOCALHOST_NETWORK://0.0.0.0:19092 + - --advertise-kafka-addr + - DOCKER_NETWORK://kafka:9092,LOCALHOST_NETWORK://127.0.0.1:19092 \ No newline at end of file diff --git a/sink-connector/tests/integration/env/mysql-master-service.yml b/sink-connector/tests/integration/env/mysql-master-service.yml new file mode 100644 index 000000000..13d92b52f --- /dev/null +++ b/sink-connector/tests/integration/env/mysql-master-service.yml @@ -0,0 +1,24 @@ +version: "2.3" + +services: + mysql-master: + container_name: mysql-master + image: docker.io/bitnami/mysql:8.0.36 + restart: "no" + expose: + - "3306" + environment: + - MYSQL_ROOT_PASSWORD=root + - MYSQL_DATABASE=test + - MYSQL_REPLICATION_MODE=master + - MYSQL_REPLICATION_USER=repl_user + - ALLOW_EMPTY_PASSWORD=yes + volumes: + - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf + - ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql + - "${CLICKHOUSE_TESTS_DIR}/_instances/share_folder:/tmp/share_folder" + healthcheck: + test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ] + interval: 15s + timeout: 5s + retries: 6 \ No newline at end of file From 30101ffa75122b58484b1d7f2896e3d2dceb82bc Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 18:35:58 +0400 Subject: [PATCH 05/44] updates --- .../tests/integration/regression.py | 6 +- .../tests/integration/tests/autocreate.py | 17 +- .../tests/integration/tests/replication.py | 38 ++ .../integration/tests/steps/clickhouse.py | 298 ++++++++++ .../tests/integration/tests/steps/mysql.py | 523 ++++++++++++++++++ .../integration/tests/steps/steps_global.py | 18 - 6 files changed, 875 insertions(+), 25 deletions(-) create mode 100644 sink-connector/tests/integration/tests/replication.py create mode 100644 sink-connector/tests/integration/tests/steps/clickhouse.py create mode 100644 sink-connector/tests/integration/tests/steps/mysql.py delete mode 100644 sink-connector/tests/integration/tests/steps/steps_global.py diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index e806e39e3..eccaa8c91 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -6,14 +6,13 @@ from testflows.core import * - append_path(sys.path, "..") +from integration.tests.steps.clickhouse import create_clickhouse_database from integration.helpers.argparser import argparser from integration.helpers.common import check_clickhouse_version from integration.helpers.common import create_cluster from integration.requirements.requirements import * -from integration.tests.steps.steps_global import * xfails = { "schema changes/table recreation with different datatypes": [ @@ -108,7 +107,7 @@ def regression( self.context.node = cluster.node("clickhouse1") with And("I create test database in ClickHouse"): - create_database(name="test") + create_clickhouse_database(name="test") modules = [ "autocreate", @@ -124,6 +123,5 @@ def regression( Feature(run=load(f"tests.{module}", "module")) - if __name__ == "__main__": regression() diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index c3003a924..8b41d3c66 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -4,7 +4,13 @@ @TestOutline -def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_table, auto_create_replicated=False): +def create_all_data_types( + self, + mysql_columns, + clickhouse_columns, + clickhouse_table, + auto_create_replicated=False, +): """Check auto-creation of replicated MySQL table which contains all supported data types. """ @@ -13,7 +19,9 @@ def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_ta mysql = self.context.cluster.node("mysql-master") init_sink_connector( - auto_create_tables=clickhouse_table[0], topics=f"SERVER5432.test.{table_name}", auto_create_replicated_tables=auto_create_replicated + auto_create_tables=clickhouse_table[0], + topics=f"SERVER5432.test.{table_name}", + auto_create_replicated_tables=auto_create_replicated, ) with Given( @@ -43,7 +51,7 @@ def create_all_data_types(self, mysql_columns, clickhouse_columns, clickhouse_ta clickhouse_table=clickhouse_table, statement="count(*)", with_final=True, - replicated=auto_create_replicated + replicated=auto_create_replicated, ) @@ -65,6 +73,7 @@ def create_all_data_types_null_table( clickhouse_table=clickhouse_table, ) + @TestFeature def create_all_data_types_null_table_replicated( self, @@ -84,6 +93,7 @@ def create_all_data_types_null_table_replicated( auto_create_replicated=True, ) + @TestFeature def create_all_data_types_not_null_table_manual( self, @@ -101,6 +111,7 @@ def create_all_data_types_not_null_table_manual( clickhouse_table=clickhouse_table, ) + @TestFeature def create_all_data_types_not_null_table_manual_replicated( self, diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py new file mode 100644 index 000000000..0f183e5c1 --- /dev/null +++ b/sink-connector/tests/integration/tests/replication.py @@ -0,0 +1,38 @@ +from testflows.core import * + + +@TestScenario +def auto_creation(self): + """Check that tables created on the source database are replicated on the destination.""" + pass + + +@TestScenario +def alters(self): + """Check that alter statements performed on the source are replicated to the destination.""" + pass + + +@TestScenario +def inserts(self): + """Check that inserts are replicated to the destination.""" + pass + + +@TestScenario +def deletes(self): + """Check that deletes are replicated to the destination.""" + pass + + +@TestScenario +def updates(self): + """Check that updates are replicated to the destination.""" + pass + + +@TestFeature +@Name("replication") +def feature(self): + """Check that replication works""" + pass diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py new file mode 100644 index 000000000..bc5943b78 --- /dev/null +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -0,0 +1,298 @@ +from integration.helpers.common import * + + +@TestStep(Then) +def drop_database(self, database_name=None, node=None, cluster=None): + """Drop ClickHouse database.""" + + if cluster is None: + cluster = "replicated_cluster" + + if database_name is None: + database_name = "test" + + if node is None: + node = self.context.cluster.node("clickhouse") + with By("executing drop database query"): + node.query( + rf"DROP DATABASE IF EXISTS {database_name} ON CLUSTER {cluster};" + ) + + +@TestStep(Then) +def check_column( + self, table_name, column_name, node=None, column_type=None, database=None +): + """Check if column exists in ClickHouse table.""" + + if database is None: + database = "test" + + if column_type is not None: + if "varchar" in column_type: + column_type = "String" + + if node is None: + node = self.context.cluster.node("clickhouse") + + if column_type is None: + select = "name" + else: + select = "name, type" + + with By(f"checking if {column_name} exists in {table_name}"): + for retry in retries(timeout=25, delay=1): + with retry: + column = node.query( + f"SELECT {select} FROM system.columns WHERE table = '{table_name}' AND name = '{column_name}' AND database = '{database}' FORMAT TabSeparated" + ) + + expected_output = ( + column_name + if column_type is None + else f"{column_name} {column_type}" + ) + + assert column.output.strip() == expected_output, error() + + + +@TestStep(Given) +def create_clickhouse_database(self, name=None, node=None): + """Create ClickHouse database.""" + if name is None: + name = "test" + + if node is None: + node = self.context.cluster.node("clickhouse") + + try: + with By(f"adding {name} database if not exists"): + drop_database(database_name=name) + + node.query( + rf"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER replicated_cluster" + ) + yield + finally: + with Finally(f"I delete {name} database if exists"): + drop_database(database_name=name) + + +@TestStep(Then) +def select( + self, + manual_output=None, + table_name=None, + statement=None, + database=None, + node=None, + with_final=False, + with_optimize=False, + sign_column="_sign", + timeout=300, +): + """SELECT statement in ClickHouse with an option to use FINAL or loop SELECT + OPTIMIZE TABLE default simple 'SELECT'""" + if node is None: + node = self.context.cluster.node("clickhouse") + if table_name is None: + table_name = "users" + if statement is None: + statement = "*" + if database is None: + database = "test" + + mysql = self.context.cluster.node("mysql-master") + mysql_output = mysql.query(f"select {statement} from {table_name}").output.strip()[ + 90: + ] + + if manual_output is None: + manual_output = mysql_output + + if with_final: + retry( + node.query, + timeout=timeout, + delay=10, + )( + f"SELECT {statement} FROM {database}.{table_name} FINAL", + message=f"{manual_output}", + ) + elif with_optimize: + for attempt in retries(count=10, timeout=100, delay=5): + with attempt: + node.query(f"OPTIMIZE TABLE {database}.{table_name} FINAL DEDUPLICATE") + + node.query( + f"SELECT {statement} FROM {database}.{table_name} where {sign_column} !=-1 FORMAT CSV", + message=f"{manual_output}", + ) + else: + retry( + node.query, + timeout=timeout, + delay=10, + )( + f"SELECT {statement} FROM {database}.{table_name} FORMAT CSV", + message=f"{manual_output}", + ) + + +@TestStep(Then) +def check_if_table_was_created( + self, table_name, database_name=None, node=None, timeout=40, message=1, replicated=False +): + """Check if table was created in ClickHouse.""" + if database_name is None: + database_name = "test" + + if node is None: + node = self.context.cluster.node("clickhouse") + + if replicated: + for node in self.context.cluster.nodes["clickhouse"]: + retry(self.context.cluster.node(node).query, timeout=timeout, delay=3)( + f"EXISTS {database_name}.{table_name}", message=f"{message}" + ) + else: + retry(node.query, timeout=timeout, delay=3)( + f"EXISTS {database_name}.{table_name}", message=f"{message}" + ) + + +@TestStep(Then) +def validate_data_in_clickhouse_table( + self, + table_name, + expected_output, + statement="*", + node=None, + database_name=None, + timeout=40, +): + """Validate data in ClickHouse table.""" + + if database_name is None: + database_name = "test" + + if node is None: + node = self.context.cluster.node("clickhouse") + + if self.context.clickhouse_table_engine == "ReplicatedReplacingMergeTree": + for node in self.context.cluster.nodes["clickhouse"]: + for retry in retries(timeout=timeout, delay=1): + with retry: + data = ( + self.context.cluster.node(node) + .query( + f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + .output.strip() + .replace('"', "") + ) + + assert ( + data == expected_output + ), f"Expected: {expected_output}, Actual: {data}" + elif self.context.clickhouse_table_engine == "ReplacingMergeTree": + for retry in retries(timeout=timeout, delay=1): + with retry: + data = ( + node.query( + f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + .output.strip() + .replace('"', "") + ) + + assert ( + data == expected_output + ), f"Expected: {expected_output}, Actual: {data}" + + else: + raise Exception("Unknown ClickHouse table engine") + + +@TestStep(Then) +def validate_rows_number( + self, table_name, expected_rows, node=None, database_name=None +): + """Validate number of rows in ClickHouse table.""" + + if database_name is None: + database_name = "test" + + if node is None: + node = self.context.cluster.node("clickhouse") + + for retry in retries(timeout=40): + with retry: + data = node.query( + f"SELECT count(*) FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + assert data.output.strip().replace('"', "") == expected_rows, error() + + +@TestStep(Then) +def verify_table_creation_in_clickhouse( + self, + table_name, + statement, + clickhouse_table_engine=(""), + timeout=300, + clickhouse_node=None, + database_name=None, + manual_output=None, + with_final=False, + with_optimize=False, + replicated=False +): + """ + Verify the creation of tables on all ClickHouse nodes where they are expected, and ensure data consistency with + MySQL. + """ + + if clickhouse_node is None: + clickhouse_node = self.context.cluster.node("clickhouse") + + if database_name is None: + database_name = "test" + + + + if replicated: + with Then("I check table creation on few nodes"): + for node in self.context.cluster.nodes["clickhouse"]: + retry(self.context.cluster.node(node).query, timeout=timeout, delay=3)( + f"EXISTS {database_name}.{table_name}", message=f"{message}" + ) + else: + with Then("I check table creation"): + retry(clickhouse_node.query, timeout=100, delay=3)( + "SHOW TABLES FROM test", message=f"{table_name}" + ) + + with Then("I check that ClickHouse table has same number of rows as MySQL table"): + select( + table_name=table_name, + manual_output=manual_output, + statement=statement, + with_final=with_final, + with_optimize=with_optimize, + timeout=timeout, + ) + if clickhouse_table_engine.startswith("Replicated"): + with Then( + "I check that ClickHouse table has same number of rows as MySQL table on the replica node if it is " + "replicted table" + ): + select( + table_name=table_name, + manual_output=manual_output, + statement=statement, + node=self.context.cluster.node("clickhouse1"), + with_final=with_final, + with_optimize=with_optimize, + timeout=timeout, + ) diff --git a/sink-connector/tests/integration/tests/steps/mysql.py b/sink-connector/tests/integration/tests/steps/mysql.py new file mode 100644 index 000000000..e451522e1 --- /dev/null +++ b/sink-connector/tests/integration/tests/steps/mysql.py @@ -0,0 +1,523 @@ +import random + +from integration.helpers.common import * +from datetime import datetime, timedelta +from testflows.core import * + + +def generate_sample_mysql_value(data_type): + """Generate a sample MySQL value for the provided datatype.""" + if data_type.startswith("DECIMAL"): + precision, scale = map( + int, data_type[data_type.index("(") + 1 : data_type.index(")")].split(",") + ) + number = round( + random.uniform(-(10 ** (precision - scale)), 10 ** (precision - scale)), + scale, + ) + return str(number) + elif data_type.startswith("DOUBLE"): + # Adjusting the range to avoid overflow, staying within a reasonable limit + return f"'{str(random.uniform(-1.7e307, 1.7e307))}'" + elif data_type == "DATE NOT NULL": + return f'\'{(datetime.today() - timedelta(days=random.randint(1, 365))).strftime("%Y-%m-%d")}\'' + elif data_type.startswith("DATETIME"): + return f'\'{(datetime.now() - timedelta(days=random.randint(1, 365))).strftime("%Y-%m-%d %H:%M:%S.%f")[:19]}\'' + elif data_type.startswith("TIME"): + if "6" in data_type: + return f'\'{(datetime.now()).strftime("%H:%M:%S.%f")[: 8 + 3]}\'' + else: + return f'\'{(datetime.now()).strftime("%H:%M:%S")}\'' + elif "INT" in data_type: + if "TINYINT" in data_type: + return str( + random.randint(0, 255) + if "UNSIGNED" in data_type + else random.randint(-128, 127) + ) + elif "SMALLINT" in data_type: + return str( + random.randint(0, 65535) + if "UNSIGNED" in data_type + else random.randint(-32768, 32767) + ) + elif "MEDIUMINT" in data_type: + return str( + random.randint(0, 16777215) + if "UNSIGNED" in data_type + else random.randint(-8388608, 8388607) + ) + elif "BIGINT" in data_type: + return str( + random.randint(0, 2**63 - 1) + if "UNSIGNED" in data_type + else random.randint(-(2**63), 2**63 - 1) + ) + else: # INT + return f'\'{str(random.randint(0, 4294967295) if "UNSIGNED" in data_type else random.randint(-2147483648, 2147483647))}\'' + elif ( + data_type.startswith("CHAR") + or data_type.startswith("VARCHAR") + or data_type.startswith("TEXT") + ): + return "'SampleText'" + elif data_type.startswith("BLOB"): + return "'SampleBinaryData'" + elif data_type.endswith("BLOB NOT NULL"): + return "'SampleBinaryData'" + elif data_type.startswith("BINARY") or data_type.startswith("VARBINARY"): + return "'a'" + else: + return "UnknownType" + + +@TestStep(Given) +def create_mysql_database(self, node=None, database_name=None): + """Creation of MySQL database.""" + if node is None: + node = self.context.cluster.node("mysql-master") + + if database_name is None: + database_name = "test" + + try: + with Given(f"I create MySQL database {database_name}"): + node.query(rf"DROP DATABASE IF EXISTS {database_name};") + node.query(rf"CREATE DATABASE IF NOT EXISTS {database_name};") + yield + finally: + with Finally(f"I delete MySQL database {database_name}"): + node.query(rf"DROP DATABASE IF EXISTS {database_name};") + + +@TestStep(Given) +def create_mysql_table( + self, + table_name, + columns, + mysql_node=None, + clickhouse_node=None, + database_name=None, + primary_key="id", + engine=True, + partition_by_mysql=False, +): + """Create MySQL table that will be auto created in ClickHouse.""" + + if database_name is None: + database_name = "test" + + if mysql_node is None: + mysql_node = self.context.cluster.node("mysql-master") + + if clickhouse_node is None: + clickhouse_node = self.context.cluster.node("clickhouse") + + try: + key = "" + if primary_key is not None: + key = f"{primary_key} INT NOT NULL," + + with Given(f"I create MySQL table", description=name): + query = f"CREATE TABLE IF NOT EXISTS {database_name}.{table_name} ({key}{columns}" + + if primary_key is not None: + query += f", PRIMARY KEY ({primary_key}))" + else: + query += ")" + + if engine: + query += f" ENGINE = InnoDB" + + if partition_by_mysql: + query += f", PARTITION BY {partition_by_mysql}" + + query += ";" + + mysql_node.query(query) + + yield + finally: + with Finally( + "I clean up by deleting MySQL to ClickHouse replicated table", + description={name}, + ): + mysql_node.query(f"DROP TABLE IF EXISTS {database_name}.{table_name};") + clickhouse_node.query( + f"DROP TABLE IF EXISTS {database_name}.{table_name} ON CLUSTER replicated_cluster;" + ) + + +@TestStep +def create_mysql_to_clickhouse_replicated_table( + self, + name, + mysql_columns, + clickhouse_table_engine, + database_name=None, + clickhouse_columns=None, + mysql_node=None, + clickhouse_node=None, + version_column="_version", + sign_column="_sign", + primary_key="id", + partition_by=None, + engine=True, + partition_by_mysql=False, +): + """Create MySQL to ClickHouse replicated table.""" + if database_name is None: + database_name = "test" + + if mysql_node is None: + mysql_node = self.context.cluster.node("mysql-master") + + if clickhouse_node is None: + clickhouse_node = self.context.cluster.node("clickhouse") + + try: + with Given(f"I create MySQL table", description=name): + query = f"CREATE TABLE IF NOT EXISTS {database_name}.{name} (id INT NOT NULL,{mysql_columns}" + + if primary_key is not None: + query += f", PRIMARY KEY ({primary_key}))" + else: + query += ")" + + if engine: + query += f" ENGINE = InnoDB" + + if partition_by_mysql: + query += f", PARTITION BY {partition_by_mysql}" + + query += ";" + + mysql_node.query(query) + + yield + finally: + with Finally( + "I clean up by deleting MySQL to CH replicated table", description={name} + ): + mysql_node.query(f"DROP TABLE IF EXISTS {database_name}.{name};") + clickhouse_node.query( + f"DROP TABLE IF EXISTS {database_name}.{name} ON CLUSTER replicated_cluster;" + ) + time.sleep(5) + + +@TestStep(Given) +def create_table_with_no_primary_key(self, table_name, clickhouse_table_engine): + """Create MySQL table without primary key.""" + + with By(f"creating a {table_name} table without primary key"): + create_mysql_to_clickhouse_replicated_table( + name=f"{table_name}_no_primary_key", + mysql_columns="x INT NOT NULL", + clickhouse_columns="x Int32", + clickhouse_table_engine=clickhouse_table_engine, + primary_key=None, + ) + + +@TestStep(Given) +def create_table_with_no_engine(self, table_name, clickhouse_table_engine): + """Create MySQL table without engine.""" + + with By(f"creating a {table_name} table without engine"): + create_mysql_to_clickhouse_replicated_table( + name=f"{table_name}_no_engine", + mysql_columns="x INT NOT NULL", + clickhouse_columns="x Int32", + clickhouse_table_engine=clickhouse_table_engine, + engine=False, + ) + + +@TestStep(Given) +def create_table_with_primary_key_and_engine(self, table_name, clickhouse_table_engine): + """Create MySQL table with primary key and with engine.""" + + with By(f"creating a {table_name} table with primary key and with engine"): + create_mysql_to_clickhouse_replicated_table( + name=f"{table_name}", + mysql_columns="x INT NOT NULL", + clickhouse_columns="x Int32", + clickhouse_table_engine=clickhouse_table_engine, + ) + + +@TestStep(Given) +def create_table_with_no_engine_and_no_primary_key( + self, table_name, clickhouse_table_engine +): + """Create MySQL table without engine and without primary key.""" + + with By(f"creating a {table_name} table without engine and without primary key"): + create_mysql_to_clickhouse_replicated_table( + name=f"{table_name}_no_engine_no_primary_key", + mysql_columns="x INT NOT NULL", + clickhouse_columns="x Int32", + clickhouse_table_engine=clickhouse_table_engine, + primary_key=None, + engine=False, + ) + + +@TestStep(Given) +def create_tables(self, table_name, clickhouse_table_engine="ReplacingMergeTree"): + """Create different types of replicated tables.""" + + with Given("I set the table names"): + tables_list = [ + f"{table_name}", + f"{table_name}_no_primary_key", + f"{table_name}_no_engine", + f"{table_name}_no_engine_no_primary_key", + ] + + with And( + "I create MySQL to ClickHouse replicated table with primary key and with engine" + ): + create_table_with_primary_key_and_engine( + table_name=table_name, clickhouse_table_engine=clickhouse_table_engine + ) + + with And( + "I create MySQL to ClickHouse replicated table without primary key and with engine" + ): + create_table_with_no_primary_key( + table_name=table_name, clickhouse_table_engine=clickhouse_table_engine + ) + + with And( + "I create MySQL to ClickHouse replicated table with primary key and without engine" + ): + create_table_with_no_engine( + table_name=table_name, clickhouse_table_engine=clickhouse_table_engine + ) + + with And( + "I create MySQL to ClickHouse replicated table without primary key and without engine" + ): + create_table_with_no_engine_and_no_primary_key( + table_name=table_name, clickhouse_table_engine=clickhouse_table_engine + ) + + return tables_list + + +@TestStep(When) +def insert(self, table_name, values, node=None, database_name=None): + """Insert data into MySQL table.""" + if database_name is None: + database_name = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + with When("I insert data into MySQL table"): + node.query(f"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});") + + +@TestStep(Given) +def insert_precondition_rows( + self, + first_insert_id, + last_insert_id, + table_name, + insert_values=None, + node=None, +): + """Insert some controlled interval of ID's in MySQL table.""" + if insert_values is None: + insert_values = "({x},2,'a','b')" + + if node is None: + node = self.context.cluster.node("mysql-master") + + with Given( + f"I insert {first_insert_id - last_insert_id} rows of data in MySQL table" + ): + for i in range(first_insert_id, last_insert_id + 1): + node.query(f"INSERT INTO `{table_name}` VALUES {insert_values}".format(x=i)) + + +@TestStep(When) +def complex_insert( + self, + table_name, + values, + start_id=1, + start_value=1, + node=None, + partitions=101, + parts_per_partition=1, + block_size=1, + exitcode=True, +): + """Insert data having specified number of partitions and parts.""" + if node is None: + node = self.context.cluster.node("mysql-master") + + x = start_id + y = start_value + + insert_values_1 = ",".join( + f"{values[0]}".format(x=x, y=y) + for x in range(start_id, partitions + start_id) + for y in range(start_value, block_size * parts_per_partition + start_value) + ) + + if exitcode: + retry( + node.query, + timeout=300, + delay=10, + )(f"INSERT INTO {table_name} VALUES {insert_values_1}", exitcode=0) + else: + retry( + node.query, + timeout=300, + delay=10, + )(f"INSERT INTO {table_name} VALUES {insert_values_1}") + + +@TestStep(When) +def delete_rows( + self, + table_name, + first_delete_id=None, + last_delete_id=None, + condition=None, + row_delete=False, + multiple=False, + no_checks=False, + check=False, + delay=False, +): + """ + Test step to delete rows from MySQL table. + """ + mysql = self.context.cluster.node("mysql-master") + + if row_delete: + if multiple: + command = ";".join( + [f"DELETE FROM {table_name} WHERE {i}" for i in condition] + ) + else: + command = f"DELETE FROM {table_name} WHERE {condition}" + r = mysql.query(command, no_checks=no_checks) + if check: + for attempt in retries(delay=0.1, timeout=30): + with attempt: + with Then("I check rows are deleted"): + check_result = mysql.query( + f"SELECT count() FROM {table_name} WHERE {condition}" + ) + assert check_result.output == "0", error() + if delay: + time.sleep(delay) + return r + else: + with Given( + f"I delete {last_delete_id - first_delete_id} rows of data in MySQL table" + ): + for i in range(first_delete_id, last_delete_id): + mysql.query(f"DELETE FROM {table_name} WHERE id={i}") + + +@TestStep(When) +def update( + self, + table_name, + first_update_id=None, + last_update_id=None, + condition=None, + row_update=False, + multiple=False, + no_checks=False, + check=False, + delay=False, +): + """Update query step for MySQL table.""" + mysql = self.context.cluster.node("mysql-master") + + if row_update: + if multiple: + command = ";".join( + [f"UPDATE {table_name} SET x=x+10000 WHERE {i}" for i in condition] + ) + else: + command = f"UPDATE {table_name} SET x=x+10000 WHERE {condition}" + r = mysql.query(command, no_checks=no_checks) + if check: + for attempt in retries(delay=0.1, timeout=30): + with attempt: + with Then("I check rows are deleted"): + check_result = mysql.query( + f"SELECT count() FROM {table_name} WHERE {condition}" + ) + assert check_result.output == "0", error() + if delay: + time.sleep(delay) + return r + else: + with Given( + f"I update {last_update_id - first_update_id} rows of data in MySQL table" + ): + for i in range(first_update_id, last_update_id): + mysql.query(f"UPDATE {table_name} SET k=k+5 WHERE id={i};") + + +@TestStep(When) +def concurrent_queries( + self, + table_name, + first_insert_number, + last_insert_number, + first_insert_id, + last_insert_id, + first_delete_id, + last_delete_id, + first_update_id, + last_update_id, +): + """Insert, update, delete for concurrent queries.""" + + with Given("I insert block of precondition rows"): + insert_precondition_rows( + table_name=table_name, + first_insert_id=first_insert_number, + last_insert_id=last_insert_number, + ) + + with When("I start concurrently insert, update and delete queries in MySQL table"): + By( + "inserting data in MySQL table", + test=insert_precondition_rows, + parallel=True, + )( + first_insert_id=first_insert_id, + last_insert_id=last_insert_id, + table_name=table_name, + ) + By( + "deleting data in MySQL table", + test=delete_rows, + parallel=True, + )( + first_delete_id=first_delete_id, + last_delete_id=last_delete_id, + table_name=table_name, + ) + By( + "updating data in MySQL table", + test=update, + parallel=True, + )( + first_update_id=first_update_id, + last_update_id=last_update_id, + table_name=table_name, + ) diff --git a/sink-connector/tests/integration/tests/steps/steps_global.py b/sink-connector/tests/integration/tests/steps/steps_global.py deleted file mode 100644 index 228737428..000000000 --- a/sink-connector/tests/integration/tests/steps/steps_global.py +++ /dev/null @@ -1,18 +0,0 @@ -from integration.helpers.common import * - - -@TestStep(Given) -def create_database(self, name="test", node=None): - """Create ClickHouse database.""" - if node is None: - node = self.context.cluster.node("clickhouse") - - try: - with By(f"adding {name} database if not exists"): - node.query( - f"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER replicated_cluster" - ) - yield - finally: - with Finally(f"I delete {name} database if exists"): - node.query(f"DROP DATABASE IF EXISTS {name} ON CLUSTER replicated_cluster;") From ec6c5642efd06bfe1792f7f54a1906dc263d83ec Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 18:50:39 +0400 Subject: [PATCH 06/44] updates --- .../integration/tests/steps/clickhouse.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index bc5943b78..227f530f5 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -14,9 +14,7 @@ def drop_database(self, database_name=None, node=None, cluster=None): if node is None: node = self.context.cluster.node("clickhouse") with By("executing drop database query"): - node.query( - rf"DROP DATABASE IF EXISTS {database_name} ON CLUSTER {cluster};" - ) + node.query(rf"DROP DATABASE IF EXISTS {database_name} ON CLUSTER {cluster};") @TestStep(Then) @@ -53,8 +51,9 @@ def check_column( else f"{column_name} {column_type}" ) - assert column.output.strip() == expected_output, error() - + assert ( + column.output.strip() == expected_output + ), f"expected {expected_output} but got {column.output.strip()}" @TestStep(Given) @@ -141,7 +140,13 @@ def select( @TestStep(Then) def check_if_table_was_created( - self, table_name, database_name=None, node=None, timeout=40, message=1, replicated=False + self, + table_name, + database_name=None, + node=None, + timeout=40, + message=1, + replicated=False, ): """Check if table was created in ClickHouse.""" if database_name is None: @@ -170,6 +175,7 @@ def validate_data_in_clickhouse_table( node=None, database_name=None, timeout=40, + replicated=False, ): """Validate data in ClickHouse table.""" @@ -179,7 +185,7 @@ def validate_data_in_clickhouse_table( if node is None: node = self.context.cluster.node("clickhouse") - if self.context.clickhouse_table_engine == "ReplicatedReplacingMergeTree": + if replicated: for node in self.context.cluster.nodes["clickhouse"]: for retry in retries(timeout=timeout, delay=1): with retry: @@ -195,7 +201,7 @@ def validate_data_in_clickhouse_table( assert ( data == expected_output ), f"Expected: {expected_output}, Actual: {data}" - elif self.context.clickhouse_table_engine == "ReplacingMergeTree": + else: for retry in retries(timeout=timeout, delay=1): with retry: data = ( @@ -210,9 +216,6 @@ def validate_data_in_clickhouse_table( data == expected_output ), f"Expected: {expected_output}, Actual: {data}" - else: - raise Exception("Unknown ClickHouse table engine") - @TestStep(Then) def validate_rows_number( @@ -246,11 +249,10 @@ def verify_table_creation_in_clickhouse( manual_output=None, with_final=False, with_optimize=False, - replicated=False + replicated=False, ): """ - Verify the creation of tables on all ClickHouse nodes where they are expected, and ensure data consistency with - MySQL. + Verify the creation of tables on all ClickHouse nodes where they are expected, and ensure data consistency with MySQL. """ if clickhouse_node is None: @@ -259,8 +261,6 @@ def verify_table_creation_in_clickhouse( if database_name is None: database_name = "test" - - if replicated: with Then("I check table creation on few nodes"): for node in self.context.cluster.nodes["clickhouse"]: From 333e93773f69b5731d1567aa292f6277f2312414 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 19:13:32 +0400 Subject: [PATCH 07/44] add steps --- .../integration/tests/steps/mysql/alters.py | 199 ++++++++++++++++++ .../integration/tests/steps/mysql/deletes.py | 27 +++ .../tests/steps/{ => mysql}/mysql.py | 0 .../integration/tests/steps/mysql/updates.py | 16 ++ 4 files changed, 242 insertions(+) create mode 100644 sink-connector/tests/integration/tests/steps/mysql/alters.py create mode 100644 sink-connector/tests/integration/tests/steps/mysql/deletes.py rename sink-connector/tests/integration/tests/steps/{ => mysql}/mysql.py (100%) create mode 100644 sink-connector/tests/integration/tests/steps/mysql/updates.py diff --git a/sink-connector/tests/integration/tests/steps/mysql/alters.py b/sink-connector/tests/integration/tests/steps/mysql/alters.py new file mode 100644 index 000000000..ab1f2a2f4 --- /dev/null +++ b/sink-connector/tests/integration/tests/steps/mysql/alters.py @@ -0,0 +1,199 @@ +from integration.helpers.common import * + + +@TestStep(When) +def add_column( + self, + table_name, + column_name="new_col", + column_type="varchar(255)", + node=None, + database=None, +): + """ADD COLUMN""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};" + ) + + +@TestStep(When) +def rename_column( + self, + table_name, + column_name="new_col", + new_column_name="new_column_name", + node=None, + database=None, +): + """RENAME COLUMN""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};" + ) + + +@TestStep(When) +def change_column( + self, + table_name, + column_name="new_col", + new_column_name="new_column_name", + new_column_type="varchar(255)", + node=None, + database=None, +): + """CHANGE COLUMN""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};" + ) + + +@TestStep(When) +def modify_column( + self, + table_name, + column_name="new_col", + new_column_type="varchar(255)", + node=None, + database=None, +): + """MODIFY COLUMN""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};" + ) + + +@TestStep(When) +def drop_column(self, table_name, column_name="new_col", node=None, database=None): + """DROP COLUMN""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query(rf"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};") + + +@TestStep(When) +def add_modify_drop_column( + self, table_name, column_name="new_col", new_column_type="INT", node=None +): + """ADD MODIFY DROP COLUMN in parallel.""" + if node is None: + node = self.context.cluster.node("mysql-master") + + By(f"add column {column_name}", test=add_column, parallel=True)( + node=node, + table_name=table_name, + column_name=column_name, + ) + + By(f"modify column {column_name}", test=modify_column, parallel=True)( + node=node, + table_name=table_name, + column_name=column_name, + new_column_type=new_column_type, + ) + + By(f"drop column {column_name}", test=drop_column, parallel=True)( + node=node, + table_name=table_name, + column_name=column_name, + ) + + join() + + +@TestStep(When) +def add_column_null_not_null( + self, + table_name, + column_name, + column_type, + is_null=False, + node=None, + database=None, +): + """ADD COLUMN NULL/NOT NULL""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + null_not_null = "NOT NULL" if not is_null else "NULL" + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};" + ) + + +@TestStep(When) +def add_column_default( + self, + table_name, + column_name, + column_type, + default_value, + node=None, + database=None, +): + """ADD COLUMN DEFAULT""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};" + ) + + +@TestStep(When) +def add_primary_key(self, table_name, column_name, node=None, database=None): + """ADD PRIMARY KEY""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query( + rf"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});" + ) + + +@TestStep(When) +def drop_primary_key(self, table_name, node=None, database=None): + """DROP PRIMARY KEY""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query(rf"ALTER TABLE {database}.\`{table_name}\` DROP PRIMARY KEY;") diff --git a/sink-connector/tests/integration/tests/steps/mysql/deletes.py b/sink-connector/tests/integration/tests/steps/mysql/deletes.py new file mode 100644 index 000000000..47ebf508e --- /dev/null +++ b/sink-connector/tests/integration/tests/steps/mysql/deletes.py @@ -0,0 +1,27 @@ +from integration.helpers.common import * + + +@TestStep(Given) +def delete(self, table, condition=None, database=None, node=None): + """Execute the DELETE query in MySQL to deleted existing records in a table.""" + if node is None: + node = self.context.cluster.node("mysql-master") + + query = rf"DELETE FROM {database}.{table} WHERE {condition};" + + if condition is not None: + query += f" WHERE {condition}" + + with By("executing DELETE query"): + node.query(query) + + +@TestStep +def delete_all_records(self, table_name, database=None, node=None): + """Delete all records from a MySQL table.""" + if node is None: + node = self.context.cluster.node("mysql-master") + + with By("executing DELETE query"): + delete(node=node, table=table_name, database=database, condition=None) + node.query(rf"DELETE FROM {database}.{table_name};") diff --git a/sink-connector/tests/integration/tests/steps/mysql.py b/sink-connector/tests/integration/tests/steps/mysql/mysql.py similarity index 100% rename from sink-connector/tests/integration/tests/steps/mysql.py rename to sink-connector/tests/integration/tests/steps/mysql/mysql.py diff --git a/sink-connector/tests/integration/tests/steps/mysql/updates.py b/sink-connector/tests/integration/tests/steps/mysql/updates.py new file mode 100644 index 000000000..ac9de2f16 --- /dev/null +++ b/sink-connector/tests/integration/tests/steps/mysql/updates.py @@ -0,0 +1,16 @@ +from integration.helpers.common import * + + +@TestStep(Given) +def update(self, table_name, database=None, node=None, condition=None, set=None): + """Update records in MySQL table.""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + query = rf"UPDATE {database}.{table_name} SET {set} WHERE {condition};" + + with By("executing UPDATE query"): + node.query(query) From d86ac90c40391606843d0496b678b0fe82fde86c Mon Sep 17 00:00:00 2001 From: Selfeer Date: Wed, 24 Jul 2024 19:14:32 +0400 Subject: [PATCH 08/44] update description --- sink-connector/tests/integration/tests/replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index 0f183e5c1..b00c7d0a6 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -34,5 +34,5 @@ def updates(self): @TestFeature @Name("replication") def feature(self): - """Check that replication works""" + """Check that actions performed on the source database are replicated on the destination database.""" pass From ad22a1d9213d3d896e6c5955c627bf72274f9f52 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Thu, 25 Jul 2024 15:43:55 +0400 Subject: [PATCH 09/44] update configuration to be customizable --- .../tests/steps/service_settings_steps.py | 157 ++++++------------ 1 file changed, 54 insertions(+), 103 deletions(-) diff --git a/sink-connector/tests/integration/tests/steps/service_settings_steps.py b/sink-connector/tests/integration/tests/steps/service_settings_steps.py index bda3f872e..17b9a4e14 100644 --- a/sink-connector/tests/integration/tests/steps/service_settings_steps.py +++ b/sink-connector/tests/integration/tests/steps/service_settings_steps.py @@ -1,4 +1,4 @@ -from integration.requirements.requirements import * +import json from integration.helpers.common import * @@ -7,73 +7,68 @@ def init_sink_connector( self, node=None, - auto_create_tables="auto", + url="clickhouse-sink-connector-kafka", + auto_create_tables=True, auto_create_replicated_tables=False, - topics="SERVER5432.sbtest.sbtest1,SERVER5432.test.users1,SERVER5432.test.users2,SERVER5432.test.users3, SERVER5432.test.users", + topics="SERVER5432.sbtest.sbtest1,SERVER5432.test.users1,SERVER5432.test.users2,SERVER5432.test.users3, " + "SERVER5432.test.users", + update=None, ): """ - Initialize sink connector. + Initialize sink connector with custom configuration. """ if node is None: node = self.context.cluster.node("bash-tools") - if auto_create_replicated_tables: - auto_create_replicated_tables = "true" - else: - auto_create_replicated_tables = "false" - if auto_create_tables == "auto": - auto_create_tables_local = "true" + auto_create_tables = True else: - auto_create_tables_local = "false" + auto_create_tables = False - # "topics": "SERVER5432.test.users", - sink_settings_transfer_command_confluent = ( - """cat </dev/null | jq ." ) @@ -105,50 +100,6 @@ def init_debezium_connector(self, node=None): if node is None: node = self.context.cluster.node("bash-tools") - debezium_settings_transfer_command_apicurio = """cat < Date: Thu, 25 Jul 2024 15:45:43 +0400 Subject: [PATCH 10/44] rename files --- sink-connector/tests/integration/tests/autocreate.py | 2 +- sink-connector/tests/integration/tests/columns_inconsistency.py | 2 +- sink-connector/tests/integration/tests/consistency.py | 2 +- sink-connector/tests/integration/tests/deduplication.py | 2 +- sink-connector/tests/integration/tests/delete.py | 2 +- sink-connector/tests/integration/tests/insert.py | 2 +- sink-connector/tests/integration/tests/multiple_tables.py | 2 +- sink-connector/tests/integration/tests/partition_limits.py | 2 +- sink-connector/tests/integration/tests/primary_keys.py | 2 +- sink-connector/tests/integration/tests/sanity.py | 2 +- sink-connector/tests/integration/tests/schema_changes.py | 2 +- .../{service_settings_steps.py => service_configurations.py} | 0 sink-connector/tests/integration/tests/sysbench.py | 2 +- sink-connector/tests/integration/tests/truncate.py | 2 +- sink-connector/tests/integration/tests/types.py | 2 +- sink-connector/tests/integration/tests/update.py | 2 +- sink-connector/tests/integration/tests/virtual_columns.py | 2 +- 17 files changed, 16 insertions(+), 16 deletions(-) rename sink-connector/tests/integration/tests/steps/{service_settings_steps.py => service_configurations.py} (100%) diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index 8b41d3c66..ba14527d0 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -1,5 +1,5 @@ from integration.tests.steps.sql import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.statements import * diff --git a/sink-connector/tests/integration/tests/columns_inconsistency.py b/sink-connector/tests/integration/tests/columns_inconsistency.py index 30af74513..7478a6258 100644 --- a/sink-connector/tests/integration/tests/columns_inconsistency.py +++ b/sink-connector/tests/integration/tests/columns_inconsistency.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/consistency.py b/sink-connector/tests/integration/tests/consistency.py index f3644d4df..edc28b2d4 100644 --- a/sink-connector/tests/integration/tests/consistency.py +++ b/sink-connector/tests/integration/tests/consistency.py @@ -2,7 +2,7 @@ from itertools import combinations from testflows.connect import Shell from integration.tests.steps.sql import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/deduplication.py b/sink-connector/tests/integration/tests/deduplication.py index 73343dc41..f6412cebf 100644 --- a/sink-connector/tests/integration/tests/deduplication.py +++ b/sink-connector/tests/integration/tests/deduplication.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/delete.py b/sink-connector/tests/integration/tests/delete.py index e83749f59..083fd018c 100644 --- a/sink-connector/tests/integration/tests/delete.py +++ b/sink-connector/tests/integration/tests/delete.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/insert.py b/sink-connector/tests/integration/tests/insert.py index 372e18757..8bc9c3445 100644 --- a/sink-connector/tests/integration/tests/insert.py +++ b/sink-connector/tests/integration/tests/insert.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/multiple_tables.py b/sink-connector/tests/integration/tests/multiple_tables.py index e110b0431..a679bb434 100644 --- a/sink-connector/tests/integration/tests/multiple_tables.py +++ b/sink-connector/tests/integration/tests/multiple_tables.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/partition_limits.py b/sink-connector/tests/integration/tests/partition_limits.py index 4a27e213c..41c60a8cf 100644 --- a/sink-connector/tests/integration/tests/partition_limits.py +++ b/sink-connector/tests/integration/tests/partition_limits.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/primary_keys.py b/sink-connector/tests/integration/tests/primary_keys.py index 804689668..6dd1b80a3 100644 --- a/sink-connector/tests/integration/tests/primary_keys.py +++ b/sink-connector/tests/integration/tests/primary_keys.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/sanity.py b/sink-connector/tests/integration/tests/sanity.py index 5ee5cc962..f7590ce92 100644 --- a/sink-connector/tests/integration/tests/sanity.py +++ b/sink-connector/tests/integration/tests/sanity.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/schema_changes.py b/sink-connector/tests/integration/tests/schema_changes.py index c38d56c8d..d6a9d3187 100644 --- a/sink-connector/tests/integration/tests/schema_changes.py +++ b/sink-connector/tests/integration/tests/schema_changes.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/steps/service_settings_steps.py b/sink-connector/tests/integration/tests/steps/service_configurations.py similarity index 100% rename from sink-connector/tests/integration/tests/steps/service_settings_steps.py rename to sink-connector/tests/integration/tests/steps/service_configurations.py diff --git a/sink-connector/tests/integration/tests/sysbench.py b/sink-connector/tests/integration/tests/sysbench.py index 2c1c1758f..b0c4dd852 100644 --- a/sink-connector/tests/integration/tests/sysbench.py +++ b/sink-connector/tests/integration/tests/sysbench.py @@ -1,6 +1,6 @@ from datetime import datetime from integration.tests.steps.sql import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestScenario diff --git a/sink-connector/tests/integration/tests/truncate.py b/sink-connector/tests/integration/tests/truncate.py index d0720964a..248eefa46 100644 --- a/sink-connector/tests/integration/tests/truncate.py +++ b/sink-connector/tests/integration/tests/truncate.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/types.py b/sink-connector/tests/integration/tests/types.py index 4e89a7a1a..fd8d8ba76 100644 --- a/sink-connector/tests/integration/tests/types.py +++ b/sink-connector/tests/integration/tests/types.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/update.py b/sink-connector/tests/integration/tests/update.py index 5f6aa822d..d247ede54 100644 --- a/sink-connector/tests/integration/tests/update.py +++ b/sink-connector/tests/integration/tests/update.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/virtual_columns.py b/sink-connector/tests/integration/tests/virtual_columns.py index ece24f2be..262da0141 100644 --- a/sink-connector/tests/integration/tests/virtual_columns.py +++ b/sink-connector/tests/integration/tests/virtual_columns.py @@ -1,6 +1,6 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_settings_steps import * +from integration.tests.steps.service_configurations import * @TestOutline From c620569023b724b1bdd878d8c8a5869403d6a684 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Thu, 25 Jul 2024 17:26:19 +0400 Subject: [PATCH 11/44] update the structure --- .../tests/integration/regression.py | 58 +++++++++--- .../tests/integration/tests/autocreate.py | 20 ++-- .../integration/tests/check_replication.py | 93 +++++++++++++++++++ .../tests/columns_inconsistency.py | 20 ++-- .../tests/integration/tests/deduplication.py | 16 ++-- .../tests/integration/tests/delete.py | 22 ++--- .../tests/integration/tests/insert.py | 20 ++-- .../tests/integration/tests/primary_keys.py | 18 ++-- .../tests/integration/tests/replication.py | 38 -------- .../tests/integration/tests/steps/sql.py | 2 +- .../integration/tests/steps/statements.py | 81 ++++++++++++++++ .../tests/integration/tests/truncate.py | 24 ++--- .../integration/tests/virtual_columns.py | 16 ++-- 13 files changed, 283 insertions(+), 145 deletions(-) create mode 100644 sink-connector/tests/integration/tests/check_replication.py delete mode 100644 sink-connector/tests/integration/tests/replication.py diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index eccaa8c91..14ac31ec4 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -109,18 +109,52 @@ def regression( with And("I create test database in ClickHouse"): create_clickhouse_database(name="test") - modules = [ - "autocreate", - "insert", - "delete", - "truncate", - "deduplication", - "primary_keys", - "virtual_columns", - "columns_inconsistency", - ] - for module in modules: - Feature(run=load(f"tests.{module}", "module")) + with Pool(1) as executor: + Feature( + run=load("tests.autocreate", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.insert", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.delete", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.truncate", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.deduplication", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.primary_keys", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.virtual_columns", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.columns_inconsistency", "feature"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.check_replication", "feature"), + parallel=True, + executor=executor, + ) if __name__ == "__main__": diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index ba14527d0..a73c94b87 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -55,7 +55,7 @@ def create_all_data_types( ) -@TestFeature +@TestScenario def create_all_data_types_null_table( self, mysql_columns=all_mysql_datatypes, @@ -74,7 +74,7 @@ def create_all_data_types_null_table( ) -@TestFeature +@TestScenario def create_all_data_types_null_table_replicated( self, mysql_columns=all_mysql_datatypes, @@ -94,7 +94,7 @@ def create_all_data_types_null_table_replicated( ) -@TestFeature +@TestScenario def create_all_data_types_not_null_table_manual( self, mysql_columns=all_nullable_mysql_datatypes, @@ -112,7 +112,7 @@ def create_all_data_types_not_null_table_manual( ) -@TestFeature +@TestScenario def create_all_data_types_not_null_table_manual_replicated( self, mysql_columns=all_nullable_mysql_datatypes, @@ -131,22 +131,18 @@ def create_all_data_types_not_null_table_manual_replicated( ) -@TestModule +@TestFeature @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_TableSchemaCreation_AutoCreate( "1.0" ) ) @Name("autocreate") -def module(self): +def feature(self): """Verify correct replication of all supported MySQL data types.""" with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/check_replication.py b/sink-connector/tests/integration/tests/check_replication.py new file mode 100644 index 000000000..8f3c48bdb --- /dev/null +++ b/sink-connector/tests/integration/tests/check_replication.py @@ -0,0 +1,93 @@ +from testflows.core import * +from integration.tests.steps.mysql.mysql import * +from integration.tests.steps.clickhouse import * +from integration.tests.steps.service_configurations import ( + init_sink_connector, + init_debezium_connector, +) +from integration.tests.steps.statements import ( + all_ch_datatypes_dict, + all_mysql_datatypes_dict, +) + + +@TestOutline +def auto_create_table( + self, columns, database_name="test", node=None, table_name=None, replicate=False +): + """Check that tables created on the source database are replicated on the destination.""" + + if table_name is None: + table_name = "table_" + getuid() + + if node is None: + node = self.context.cluster.node("mysql-master") + + with Given("I initialize sink connector for the given database and table"): + init_sink_connector( + auto_create_tables=True, + topics=f"SERVER5432.{database_name}.{table_name}", + auto_create_replicated_tables=replicate, + ) + + with And("I create a table on the source database"): + create_mysql_table( + table_name=table_name, + database_name=database_name, + mysql_node=node, + columns=columns, + ) + + with When("I insert values into the table"): + insert(table_name=table_name, values=generate_sample_mysql_value(columns)) + + with Then("I check that the table is replicated on the destination database"): + check_if_table_was_created(table_name=table_name) + + +@TestScenario +def check_auto_creation_all_datatypes(self): + """Check that tables created on the source database are replicated on the destination.""" + datatypes = [ + all_mysql_datatypes_dict[datatype] for datatype in all_mysql_datatypes_dict + ] + + for datatype in datatypes: + auto_create_table( + columns=datatype, + ) + + +@TestScenario +def alters(self): + """Check that alter statements performed on the source are replicated to the destination.""" + note(1) + + +@TestScenario +def inserts(self): + """Check that inserts are replicated to the destination.""" + note(1) + + +@TestScenario +def deletes(self): + """Check that deletes are replicated to the destination.""" + note(1) + + +@TestScenario +def updates(self): + """Check that updates are replicated to the destination.""" + note(1) + + +@TestFeature +@Name("replication") +def feature(self): + """Check that actions performed on the source database are replicated on the destination database.""" + with Given("I enable debezium and sink connectors after kafka starts up"): + init_debezium_connector() + + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/columns_inconsistency.py b/sink-connector/tests/integration/tests/columns_inconsistency.py index 7478a6258..d1d7ce427 100644 --- a/sink-connector/tests/integration/tests/columns_inconsistency.py +++ b/sink-connector/tests/integration/tests/columns_inconsistency.py @@ -38,7 +38,7 @@ def mysql_to_clickhouse_insert( ) -@TestFeature +@TestScenario def more_columns( self, input="(2,7,777)", @@ -59,7 +59,7 @@ def more_columns( ) -@TestFeature +@TestScenario def less_columns( self, input="(2,7,777)", @@ -80,7 +80,7 @@ def less_columns( ) -@TestFeature +@TestScenario def equal_columns_different_names( self, input="(2,7,777)", @@ -101,7 +101,7 @@ def equal_columns_different_names( ) -@TestFeature +@TestScenario def equal_columns_some_different_names( self, input="(2,7,777)", @@ -122,20 +122,16 @@ def equal_columns_some_different_names( ) -@TestModule +@TestFeature @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnsInconsistency("1.0") ) @Name("columns inconsistency") -def module(self): +def feature(self): """Check for different columns inconsistency.""" with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/deduplication.py b/sink-connector/tests/integration/tests/deduplication.py index f6412cebf..f4e9673cb 100644 --- a/sink-connector/tests/integration/tests/deduplication.py +++ b/sink-connector/tests/integration/tests/deduplication.py @@ -48,7 +48,7 @@ def deduplication( ) -@TestFeature +@TestScenario def deduplication_on_big_insert(self): """Check MySQL to Clickhouse connection for non-duplication data on 10 000 inserts.""" for clickhouse_table in available_clickhouse_tables: @@ -58,7 +58,7 @@ def deduplication_on_big_insert(self): ) -@TestFeature +@TestScenario def deduplication_on_many_inserts(self): """Check MySQL to Clickhouse connection for non-duplication data on big inserts.""" for clickhouse_table in available_clickhouse_tables: @@ -68,21 +68,17 @@ def deduplication_on_many_inserts(self): ) -@TestModule +@TestFeature @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Consistency_Deduplication("1.0") ) @Name("deduplication") -def module(self): +def feature(self): """MySql to ClickHouse replication tests to check for non-duplication data on big inserts.""" with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/delete.py b/sink-connector/tests/integration/tests/delete.py index 083fd018c..c1503f52a 100644 --- a/sink-connector/tests/integration/tests/delete.py +++ b/sink-connector/tests/integration/tests/delete.py @@ -43,7 +43,7 @@ def delete( ) -@TestFeature +@TestScenario def no_primary_key(self): """Check for `DELETE` with no primary key without InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -57,7 +57,7 @@ def no_primary_key(self): ) -@TestFeature +@TestScenario def no_primary_key_innodb(self): """Check for `DELETE` with no primary key with InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -85,7 +85,7 @@ def simple_primary_key(self): ) -@TestFeature +@TestScenario def simple_primary_key_innodb(self): """Check for `DELETE` with simple primary key with InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -99,7 +99,7 @@ def simple_primary_key_innodb(self): ) -@TestFeature +@TestScenario def complex_primary_key(self): """Check for `DELETE` with complex primary key without engine InnoDB.""" for clickhouse_table in available_clickhouse_tables: @@ -113,7 +113,7 @@ def complex_primary_key(self): ) -@TestFeature +@TestScenario def complex_primary_key_innodb(self): """Check for `DELETE` with complex primary key with engine InnoDB.""" for clickhouse_table in available_clickhouse_tables: @@ -127,18 +127,14 @@ def complex_primary_key_innodb(self): ) -@TestModule +@TestFeature @Requirements(RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Deletes("1.0")) @Name("delete") -def module(self): +def feature(self): """MySql to ClickHouse replication delete tests to test `DELETE` queries.""" with Given("I enable debezium connector after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/insert.py b/sink-connector/tests/integration/tests/insert.py index 8bc9c3445..98b250bce 100644 --- a/sink-connector/tests/integration/tests/insert.py +++ b/sink-connector/tests/integration/tests/insert.py @@ -37,7 +37,7 @@ def mysql_to_clickhouse_inserts( ) -@TestFeature +@TestScenario def null_default_insert( self, input="(DEFAULT,5,DEFAULT)", @@ -57,7 +57,7 @@ def null_default_insert( ) -@TestFeature +@TestScenario def null_default_insert_2( self, input="(DEFAULT,5,333)", @@ -77,7 +77,7 @@ def null_default_insert_2( ) -@TestFeature +@TestScenario def select_insert( self, input="((select 2),7,DEFAULT)", @@ -97,7 +97,7 @@ def select_insert( ) -@TestFeature +@TestScenario def select_insert_2( self, input="((select 2),7,DEFAULT)", @@ -117,19 +117,15 @@ def select_insert_2( ) -@TestModule +@TestFeature @Requirements(RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Inserts("1.0")) @Name("insert") -def module(self): +def feature(self): """Different `INSERT` tests section.""" # xfail("") with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/primary_keys.py b/sink-connector/tests/integration/tests/primary_keys.py index 6dd1b80a3..07b9e3148 100644 --- a/sink-connector/tests/integration/tests/primary_keys.py +++ b/sink-connector/tests/integration/tests/primary_keys.py @@ -51,7 +51,7 @@ def check_different_primary_keys( ) -@TestFeature +@TestScenario @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_PrimaryKey_Simple("1.0") ) @@ -70,7 +70,7 @@ def simple_primary_key(self): ) -@TestFeature +@TestScenario @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_PrimaryKey_Composite("1.0") ) @@ -89,7 +89,7 @@ def composite_primary_key(self): ) -@TestFeature +@TestScenario @Requirements(RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_PrimaryKey_No("1.0")) def no_primary_key(self): """Check replicating MySQl table without any primary key.""" @@ -107,17 +107,13 @@ def no_primary_key(self): ) -@TestModule +@TestFeature @Name("primary keys") -def module(self): +def feature(self): """MySql to ClickHouse replication simple and composite primary keys tests.""" with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py deleted file mode 100644 index b00c7d0a6..000000000 --- a/sink-connector/tests/integration/tests/replication.py +++ /dev/null @@ -1,38 +0,0 @@ -from testflows.core import * - - -@TestScenario -def auto_creation(self): - """Check that tables created on the source database are replicated on the destination.""" - pass - - -@TestScenario -def alters(self): - """Check that alter statements performed on the source are replicated to the destination.""" - pass - - -@TestScenario -def inserts(self): - """Check that inserts are replicated to the destination.""" - pass - - -@TestScenario -def deletes(self): - """Check that deletes are replicated to the destination.""" - pass - - -@TestScenario -def updates(self): - """Check that updates are replicated to the destination.""" - pass - - -@TestFeature -@Name("replication") -def feature(self): - """Check that actions performed on the source database are replicated on the destination database.""" - pass diff --git a/sink-connector/tests/integration/tests/steps/sql.py b/sink-connector/tests/integration/tests/steps/sql.py index c6a501f7d..9e89cfed3 100644 --- a/sink-connector/tests/integration/tests/steps/sql.py +++ b/sink-connector/tests/integration/tests/steps/sql.py @@ -69,7 +69,7 @@ def create_clickhouse_table( ) -@TestStep +@TestStep(Given) def create_mysql_to_clickhouse_replicated_table( self, name, diff --git a/sink-connector/tests/integration/tests/steps/statements.py b/sink-connector/tests/integration/tests/steps/statements.py index 403d0ea0d..132c0c189 100644 --- a/sink-connector/tests/integration/tests/steps/statements.py +++ b/sink-connector/tests/integration/tests/steps/statements.py @@ -110,3 +110,84 @@ f" x_binary String," f" x_varbinary String" ) + +all_mysql_datatypes_dict = { + "D4": "DECIMAL(2,1) NOT NULL", + "D5": "DECIMAL(30, 10) NOT NULL", + "Doublex": "DOUBLE NOT NULL", + "x_date": "DATE NOT NULL", + "x_datetime6": "DATETIME(6) NOT NULL", + "x_time": "TIME NOT NULL", + "x_time6": "TIME(6) NOT NULL", + "Intmin": "INT NOT NULL", + "Intmax": "INT NOT NULL", + "UIntmin": "INT UNSIGNED NOT NULL", + "UIntmax": "INT UNSIGNED NOT NULL", + "BIGIntmin": "BIGINT NOT NULL", + "BIGIntmax": "BIGINT NOT NULL", + "UBIGIntmin": "BIGINT UNSIGNED NOT NULL", + "UBIGIntmax": "BIGINT UNSIGNED NOT NULL", + "TIntmin": "TINYINT NOT NULL", + "TIntmax": "TINYINT NOT NULL", + "UTIntmin": "TINYINT UNSIGNED NOT NULL", + "UTIntmax": "TINYINT UNSIGNED NOT NULL", + "SIntmin": "SMALLINT NOT NULL", + "SIntmax": "SMALLINT NOT NULL", + "USIntmin": "SMALLINT UNSIGNED NOT NULL", + "USIntmax": "SMALLINT UNSIGNED NOT NULL", + "MIntmin": "MEDIUMINT NOT NULL", + "MIntmax": "MEDIUMINT NOT NULL", + "UMIntmin": "MEDIUMINT UNSIGNED NOT NULL", + "UMIntmax": "MEDIUMINT UNSIGNED NOT NULL", + "x_char": "CHAR(255) NOT NULL", + "x_text": "TEXT(255) NOT NULL", + "x_varchar": "VARCHAR(255) NOT NULL", + "x_Blob": "BLOB(255) NOT NULL", + "x_Mediumblob": "MEDIUMBLOB NOT NULL", + "x_Longblob": "LONGBLOB NOT NULL", + "x_binary": "BINARY NOT NULL", + "x_varbinary": "VARBINARY(4) NOT NULL", +} + +all_ch_datatypes_dict = { + "D4": "DECIMAL(2,1)", + "D5": "DECIMAL(30, 10)", + "Doublex": "Float64", + "x_date": "Date", + "x_datetime6": "String", + "x_time": "String", + "x_time6": "String", + "Intmin": "Int32", + "Intmax": "Int32", + "UIntmin": "UInt32", + "UIntmax": "UInt32", + "BIGIntmin": "UInt64", + "BIGIntmax": "UInt64", + "UBIGIntmin": "UInt64", + "UBIGIntmax": "UInt64", + "TIntmin": "Int8", + "TIntmax": "Int8", + "UTIntmin": "UInt8", + "UTIntmax": "UInt8", + "SIntmin": "Int16", + "SIntmax": "Int16", + "USIntmin": "UInt16", + "USIntmax": "UInt16", + "MIntmin": "Int32", + "MIntmax": "Int32", + "UMIntmin": "UInt32", + "UMIntmax": "UInt32", + "x_char": "LowCardinality(String)", + "x_text": "String", + "x_varchar": "String", + "x_Blob": "String", + "x_Mediumblob": "String", + "x_Longblob": "String", + "x_binary": "String", + "x_varbinary": "String", +} + +datatypes_to_add = ( + "TINYBLOB, TINYTEXT, SET, BIT, BOOL, BOOLEAN,INTEGER, FLOAT(size, d), FLOAT(p), DOUBLE PRECISION(" + "size, d), DEC, TIMESTAMP, YEAR, ENUM(val1, val2, val3, ...)" +) diff --git a/sink-connector/tests/integration/tests/truncate.py b/sink-connector/tests/integration/tests/truncate.py index 248eefa46..7922e94bc 100644 --- a/sink-connector/tests/integration/tests/truncate.py +++ b/sink-connector/tests/integration/tests/truncate.py @@ -50,7 +50,7 @@ def truncate( ) -@TestFeature +@TestScenario def no_primary_key(self): """Check for `DELETE` with no primary key without InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -64,7 +64,7 @@ def no_primary_key(self): ) -@TestFeature +@TestScenario def no_primary_key_innodb(self): """Check for `DELETE` with no primary key with InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -78,7 +78,7 @@ def no_primary_key_innodb(self): ) -@TestFeature +@TestScenario def simple_primary_key(self): """Check for `DELETE` with simple primary key without InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -92,7 +92,7 @@ def simple_primary_key(self): ) -@TestFeature +@TestScenario def simple_primary_key_innodb(self): """Check for `DELETE` with simple primary key with InnoDB engine.""" for clickhouse_table in available_clickhouse_tables: @@ -106,7 +106,7 @@ def simple_primary_key_innodb(self): ) -@TestFeature +@TestScenario def complex_primary_key(self): """Check for `DELETE` with complex primary key without engine InnoDB.""" for clickhouse_table in available_clickhouse_tables: @@ -120,7 +120,7 @@ def complex_primary_key(self): ) -@TestFeature +@TestScenario def complex_primary_key_innodb(self): """Check for `DELETE` with complex primary key with engine InnoDB.""" for clickhouse_table in available_clickhouse_tables: @@ -134,16 +134,12 @@ def complex_primary_key_innodb(self): ) -@TestModule +@TestFeature @Name("truncate") -def module(self): +def feature(self): """'ALTER TRUNCATE' query tests.""" with Given("I enable debezium connector after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/sink-connector/tests/integration/tests/virtual_columns.py b/sink-connector/tests/integration/tests/virtual_columns.py index 262da0141..27f012179 100644 --- a/sink-connector/tests/integration/tests/virtual_columns.py +++ b/sink-connector/tests/integration/tests/virtual_columns.py @@ -54,7 +54,7 @@ def virtual_column_names( ) -@TestFeature +@TestScenario def virtual_column_names_default(self): """Check correctness of default virtual column names.""" for clickhouse_table in available_clickhouse_tables: @@ -63,7 +63,7 @@ def virtual_column_names_default(self): virtual_column_names(clickhouse_table=clickhouse_table) -@TestFeature +@TestScenario @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MySQLStorageEngines_ReplicatedReplacingMergeTree_DifferentVersionColumnNames( "1.0" @@ -81,17 +81,13 @@ def virtual_column_names_replicated_random(self): ) -@TestModule +@TestFeature @Name("virtual columns") -def module(self): +def feature(self): """Section to check behavior of virtual columns.""" with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + for scenario in loads(current_module(), Scenario): + scenario() From 3ce352f04c1bf429ef2e6620195163d1ea94f5f7 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 26 Jul 2024 12:31:13 +0400 Subject: [PATCH 12/44] refactor --- .../tests/integration/regression.py | 57 ++++--------------- .../{check_replication.py => replication.py} | 23 +++++--- 2 files changed, 25 insertions(+), 55 deletions(-) rename sink-connector/tests/integration/tests/{check_replication.py => replication.py} (82%) diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index 14ac31ec4..ac56eacd5 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -104,57 +104,20 @@ def regression( if check_clickhouse_version("<21.4")(self): skip(reason="only supported on ClickHouse version >= 21.4") - self.context.node = cluster.node("clickhouse1") + self.context.node = cluster.node("clickhouse") with And("I create test database in ClickHouse"): create_clickhouse_database(name="test") - with Pool(1) as executor: - Feature( - run=load("tests.autocreate", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.insert", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.delete", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.truncate", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.deduplication", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.primary_keys", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.virtual_columns", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.columns_inconsistency", "feature"), - parallel=True, - executor=executor, - ) - Feature( - run=load("tests.check_replication", "feature"), - parallel=True, - executor=executor, - ) + Feature(run=load("tests.autocreate", "feature")) + Feature(run=load("tests.insert", "feature")) + Feature(run=load("tests.delete", "feature")) + Feature(run=load("tests.truncate", "feature")) + Feature(run=load("tests.deduplication", "feature")) + Feature(run=load("tests.primary_keys", "feature")) + Feature(run=load("tests.virtual_columns", "feature")) + Feature(run=load("tests.columns_inconsistency", "feature")) + Feature(run=load("tests.replication", "feature")) if __name__ == "__main__": diff --git a/sink-connector/tests/integration/tests/check_replication.py b/sink-connector/tests/integration/tests/replication.py similarity index 82% rename from sink-connector/tests/integration/tests/check_replication.py rename to sink-connector/tests/integration/tests/replication.py index 8f3c48bdb..9cc80d423 100644 --- a/sink-connector/tests/integration/tests/check_replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -13,7 +13,13 @@ @TestOutline def auto_create_table( - self, columns, database_name="test", node=None, table_name=None, replicate=False + self, + column_datatype, + column_name, + database_name="test", + node=None, + table_name=None, + replicate=False, ): """Check that tables created on the source database are replicated on the destination.""" @@ -35,11 +41,14 @@ def auto_create_table( table_name=table_name, database_name=database_name, mysql_node=node, - columns=columns, + columns=f"{column_name} {column_datatype}", ) with When("I insert values into the table"): - insert(table_name=table_name, values=generate_sample_mysql_value(columns)) + insert( + table_name=table_name, + values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", + ) with Then("I check that the table is replicated on the destination database"): check_if_table_was_created(table_name=table_name) @@ -48,13 +57,11 @@ def auto_create_table( @TestScenario def check_auto_creation_all_datatypes(self): """Check that tables created on the source database are replicated on the destination.""" - datatypes = [ - all_mysql_datatypes_dict[datatype] for datatype in all_mysql_datatypes_dict - ] - for datatype in datatypes: + for name, datatype in all_mysql_datatypes_dict.items(): auto_create_table( - columns=datatype, + column_name=name, + column_datatype=datatype, ) From a4c0a9863daed5bb4f01b234c35cd67cd1340b0a Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 26 Jul 2024 17:10:56 +0400 Subject: [PATCH 13/44] update imports --- .../tests/integration/regression.py | 2 +- .../tests/integration/tests/autocreate.py | 1 + .../tests/columns_inconsistency.py | 1 + .../tests/integration/tests/consistency.py | 1 + .../tests/integration/tests/delete.py | 1 + .../tests/integration/tests/insert.py | 1 + .../integration/tests/multiple_tables.py | 1 + .../integration/tests/partition_limits.py | 1 + .../tests/integration/tests/primary_keys.py | 1 + .../tests/integration/tests/replication.py | 82 ++++++++++++------- .../tests/integration/tests/sanity.py | 1 + .../tests/integration/tests/schema_changes.py | 1 + .../integration/tests/steps/clickhouse.py | 4 +- .../integration/tests/steps/mysql/mysql.py | 8 +- .../tests/steps/service_configurations.py | 2 +- .../tests/integration/tests/steps/sql.py | 48 ++++++++++- .../tests/integration/tests/sysbench.py | 1 + .../tests/integration/tests/truncate.py | 1 + .../tests/integration/tests/types.py | 1 + .../tests/integration/tests/update.py | 1 + .../integration/tests/virtual_columns.py | 1 + 21 files changed, 124 insertions(+), 37 deletions(-) diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index ac56eacd5..9498eb3a0 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -104,7 +104,7 @@ def regression( if check_clickhouse_version("<21.4")(self): skip(reason="only supported on ClickHouse version >= 21.4") - self.context.node = cluster.node("clickhouse") + self.context.node = cluster.node("clickhouse1") with And("I create test database in ClickHouse"): create_clickhouse_database(name="test") diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index a73c94b87..829149aec 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -1,3 +1,4 @@ +from integration.requirements.requirements import * from integration.tests.steps.sql import * from integration.tests.steps.service_configurations import * from integration.tests.steps.statements import * diff --git a/sink-connector/tests/integration/tests/columns_inconsistency.py b/sink-connector/tests/integration/tests/columns_inconsistency.py index d1d7ce427..0b915e2a1 100644 --- a/sink-connector/tests/integration/tests/columns_inconsistency.py +++ b/sink-connector/tests/integration/tests/columns_inconsistency.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/consistency.py b/sink-connector/tests/integration/tests/consistency.py index edc28b2d4..e034a39f3 100644 --- a/sink-connector/tests/integration/tests/consistency.py +++ b/sink-connector/tests/integration/tests/consistency.py @@ -3,6 +3,7 @@ from testflows.connect import Shell from integration.tests.steps.sql import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/delete.py b/sink-connector/tests/integration/tests/delete.py index c1503f52a..08b78b289 100644 --- a/sink-connector/tests/integration/tests/delete.py +++ b/sink-connector/tests/integration/tests/delete.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/insert.py b/sink-connector/tests/integration/tests/insert.py index 98b250bce..76c5a9331 100644 --- a/sink-connector/tests/integration/tests/insert.py +++ b/sink-connector/tests/integration/tests/insert.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/multiple_tables.py b/sink-connector/tests/integration/tests/multiple_tables.py index a679bb434..f37ddd246 100644 --- a/sink-connector/tests/integration/tests/multiple_tables.py +++ b/sink-connector/tests/integration/tests/multiple_tables.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/partition_limits.py b/sink-connector/tests/integration/tests/partition_limits.py index 41c60a8cf..7143cf96c 100644 --- a/sink-connector/tests/integration/tests/partition_limits.py +++ b/sink-connector/tests/integration/tests/partition_limits.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/primary_keys.py b/sink-connector/tests/integration/tests/primary_keys.py index 07b9e3148..1f2806d7b 100644 --- a/sink-connector/tests/integration/tests/primary_keys.py +++ b/sink-connector/tests/integration/tests/primary_keys.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index 9cc80d423..f3270418a 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -5,10 +5,12 @@ init_sink_connector, init_debezium_connector, ) +from integration.tests.steps.sql import generate_interesting_table_names from integration.tests.steps.statements import ( all_ch_datatypes_dict, all_mysql_datatypes_dict, ) +from integration.requirements.requirements import * @TestOutline @@ -16,7 +18,7 @@ def auto_create_table( self, column_datatype, column_name, - database_name="test", + databases, node=None, table_name=None, replicate=False, @@ -28,40 +30,61 @@ def auto_create_table( if node is None: node = self.context.cluster.node("mysql-master") + for database in databases: + with Given("I initialize sink connector for the given database and table"): + init_sink_connector( + auto_create_tables=True, + topics=f"SERVER5432.{database}.{table_name}", + auto_create_replicated_tables=replicate, + ) + + with And("I create a table on the source database"): + create_mysql_table( + table_name=table_name, + database_name=database, + mysql_node=node, + columns=f"{column_name} {column_datatype}", + ) + + with When("I insert values into the table"): + insert( + table_name=table_name, + values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", + ) + + with Then("I check that the table is replicated on the destination database"): + with Check(f"table with {column_datatype} was replicated"): + check_if_table_was_created( + database_name=database, table_name=table_name + ) - with Given("I initialize sink connector for the given database and table"): - init_sink_connector( - auto_create_tables=True, - topics=f"SERVER5432.{database_name}.{table_name}", - auto_create_replicated_tables=replicate, - ) - with And("I create a table on the source database"): - create_mysql_table( - table_name=table_name, - database_name=database_name, - mysql_node=node, - columns=f"{column_name} {column_datatype}", - ) +@TestScenario +def check_auto_creation_all_datatypes(self, databases=None, table_name=None): + """Check that tables created on the source database are replicated on the destination.""" - with When("I insert values into the table"): - insert( - table_name=table_name, - values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", - ) + if databases is None: + databases = ["test"] - with Then("I check that the table is replicated on the destination database"): - check_if_table_was_created(table_name=table_name) + for name, datatype in all_mysql_datatypes_dict.items(): + ( + Check(test=auto_create_table, name=f"auto table creation {datatype}")( + column_name=name, + column_datatype=datatype, + databases=databases, + table_name=table_name, + ) + ) @TestScenario -def check_auto_creation_all_datatypes(self): - """Check that tables created on the source database are replicated on the destination.""" +def auto_creation_different_table_names(self, databases=None): + """Check that tables with all datatypes are replicated on the destination table when tables have names with special cases.""" + table_names = generate_interesting_table_names(self.context.number_of_tables) - for name, datatype in all_mysql_datatypes_dict.items(): - auto_create_table( - column_name=name, - column_datatype=datatype, + for table_name in table_names: + Check(test=check_auto_creation_all_datatypes)( + databases=databases, table_name=table_name ) @@ -91,8 +114,11 @@ def updates(self): @TestFeature @Name("replication") -def feature(self): +def feature(self, number_of_tables=150): """Check that actions performed on the source database are replicated on the destination database.""" + + self.context.number_of_tables = number_of_tables + with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() diff --git a/sink-connector/tests/integration/tests/sanity.py b/sink-connector/tests/integration/tests/sanity.py index f7590ce92..5cd18a08e 100644 --- a/sink-connector/tests/integration/tests/sanity.py +++ b/sink-connector/tests/integration/tests/sanity.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/schema_changes.py b/sink-connector/tests/integration/tests/schema_changes.py index d6a9d3187..4fb9969ae 100644 --- a/sink-connector/tests/integration/tests/schema_changes.py +++ b/sink-connector/tests/integration/tests/schema_changes.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index 227f530f5..38b2a3eea 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -158,11 +158,11 @@ def check_if_table_was_created( if replicated: for node in self.context.cluster.nodes["clickhouse"]: retry(self.context.cluster.node(node).query, timeout=timeout, delay=3)( - f"EXISTS {database_name}.{table_name}", message=f"{message}" + rf"EXISTS {database_name}.\`{table_name}\`", message=f"{message}" ) else: retry(node.query, timeout=timeout, delay=3)( - f"EXISTS {database_name}.{table_name}", message=f"{message}" + f"EXISTS {database_name}.\`{table_name}\`", message=f"{message}" ) diff --git a/sink-connector/tests/integration/tests/steps/mysql/mysql.py b/sink-connector/tests/integration/tests/steps/mysql/mysql.py index e451522e1..1cee7e4c5 100644 --- a/sink-connector/tests/integration/tests/steps/mysql/mysql.py +++ b/sink-connector/tests/integration/tests/steps/mysql/mysql.py @@ -119,7 +119,7 @@ def create_mysql_table( key = f"{primary_key} INT NOT NULL," with Given(f"I create MySQL table", description=name): - query = f"CREATE TABLE IF NOT EXISTS {database_name}.{table_name} ({key}{columns}" + query = rf"CREATE TABLE IF NOT EXISTS {database_name}.\`{table_name}\` ({key}{columns}" if primary_key is not None: query += f", PRIMARY KEY ({primary_key}))" @@ -142,9 +142,9 @@ def create_mysql_table( "I clean up by deleting MySQL to ClickHouse replicated table", description={name}, ): - mysql_node.query(f"DROP TABLE IF EXISTS {database_name}.{table_name};") + mysql_node.query(rf"DROP TABLE IF EXISTS {database_name}.\`{table_name}\`;") clickhouse_node.query( - f"DROP TABLE IF EXISTS {database_name}.{table_name} ON CLUSTER replicated_cluster;" + rf"DROP TABLE IF EXISTS {database_name}.\`{table_name}\` ON CLUSTER replicated_cluster;" ) @@ -317,7 +317,7 @@ def insert(self, table_name, values, node=None, database_name=None): node = self.context.cluster.node("mysql-master") with When("I insert data into MySQL table"): - node.query(f"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});") + node.query(rf"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});") @TestStep(Given) diff --git a/sink-connector/tests/integration/tests/steps/service_configurations.py b/sink-connector/tests/integration/tests/steps/service_configurations.py index 17b9a4e14..3561e541a 100644 --- a/sink-connector/tests/integration/tests/steps/service_configurations.py +++ b/sink-connector/tests/integration/tests/steps/service_configurations.py @@ -22,7 +22,7 @@ def init_sink_connector( if auto_create_tables == "auto": auto_create_tables = True - else: + elif auto_create_tables == "manual": auto_create_tables = False default_config = { diff --git a/sink-connector/tests/integration/tests/steps/sql.py b/sink-connector/tests/integration/tests/steps/sql.py index 9e89cfed3..d7c6b2d96 100644 --- a/sink-connector/tests/integration/tests/steps/sql.py +++ b/sink-connector/tests/integration/tests/steps/sql.py @@ -1,8 +1,54 @@ -from integration.requirements.requirements import * +import random +import string from integration.helpers.common import * +def generate_interesting_table_names(num_names, max_length=64): + """Generate a list of interesting MySQL table names for testing.""" + reserved_keywords = [ + "Select", + "Insert", + "Update", + "Delete", + "Group", + "Where", + "Transaction", + ] + + special_chars = "_$" + utf8_chars = "áéíóúñüç" + punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_{|}~""" + + def generate_table_name(length): + """Generate a random table name of a given length.""" + allowed_chars = ( + string.ascii_letters + + string.digits + + special_chars + + utf8_chars + + punctuation + ) + return "".join(random.choice(allowed_chars) for _ in range(length)) + + table_names = set() + + # Add reserved keywords + table_names.update(reserved_keywords) + + while len(table_names) < num_names: + length = random.randint(1, max_length) + name = generate_table_name(length) + + # Ensure the name does not start with a digit + if name[0] in string.digits: + name = "_" + name + + table_names.add(f"{name}") + + return list(table_names) + + @TestStep(Given) def create_mysql_table(self, name=None, statement=None, node=None): """ diff --git a/sink-connector/tests/integration/tests/sysbench.py b/sink-connector/tests/integration/tests/sysbench.py index b0c4dd852..700637882 100644 --- a/sink-connector/tests/integration/tests/sysbench.py +++ b/sink-connector/tests/integration/tests/sysbench.py @@ -1,6 +1,7 @@ from datetime import datetime from integration.tests.steps.sql import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestScenario diff --git a/sink-connector/tests/integration/tests/truncate.py b/sink-connector/tests/integration/tests/truncate.py index 7922e94bc..6768b2e89 100644 --- a/sink-connector/tests/integration/tests/truncate.py +++ b/sink-connector/tests/integration/tests/truncate.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/types.py b/sink-connector/tests/integration/tests/types.py index fd8d8ba76..fa56f3c72 100644 --- a/sink-connector/tests/integration/tests/types.py +++ b/sink-connector/tests/integration/tests/types.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/update.py b/sink-connector/tests/integration/tests/update.py index d247ede54..459d9e52b 100644 --- a/sink-connector/tests/integration/tests/update.py +++ b/sink-connector/tests/integration/tests/update.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/virtual_columns.py b/sink-connector/tests/integration/tests/virtual_columns.py index 27f012179..919cd3725 100644 --- a/sink-connector/tests/integration/tests/virtual_columns.py +++ b/sink-connector/tests/integration/tests/virtual_columns.py @@ -1,6 +1,7 @@ from integration.tests.steps.sql import * from integration.tests.steps.statements import * from integration.tests.steps.service_configurations import * +from integration.requirements.requirements import * @TestOutline From ab309220cbe538fe70457c4e5da5e9ad0df8b20b Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 26 Jul 2024 17:12:49 +0400 Subject: [PATCH 14/44] update imports --- sink-connector/tests/integration/regression.py | 1 - sink-connector/tests/integration/tests/autocreate.py | 2 +- .../tests/integration/tests/columns_inconsistency.py | 4 ++-- sink-connector/tests/integration/tests/consistency.py | 7 ++++--- sink-connector/tests/integration/tests/deduplication.py | 3 ++- sink-connector/tests/integration/tests/delete.py | 4 ++-- sink-connector/tests/integration/tests/insert.py | 4 ++-- sink-connector/tests/integration/tests/multiple_tables.py | 4 ++-- sink-connector/tests/integration/tests/partition_limits.py | 4 ++-- sink-connector/tests/integration/tests/primary_keys.py | 4 ++-- sink-connector/tests/integration/tests/replication.py | 5 +---- sink-connector/tests/integration/tests/sanity.py | 3 +-- sink-connector/tests/integration/tests/schema_changes.py | 3 +-- sink-connector/tests/integration/tests/steps/statements.py | 1 + sink-connector/tests/integration/tests/sysbench.py | 3 +-- sink-connector/tests/integration/tests/truncate.py | 3 +-- sink-connector/tests/integration/tests/types.py | 4 ++-- sink-connector/tests/integration/tests/update.py | 4 ++-- sink-connector/tests/integration/tests/virtual_columns.py | 4 ++-- 19 files changed, 31 insertions(+), 36 deletions(-) diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index 9498eb3a0..9e4a65f4a 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -117,7 +117,6 @@ def regression( Feature(run=load("tests.primary_keys", "feature")) Feature(run=load("tests.virtual_columns", "feature")) Feature(run=load("tests.columns_inconsistency", "feature")) - Feature(run=load("tests.replication", "feature")) if __name__ == "__main__": diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index 829149aec..4698c2af4 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -1,6 +1,6 @@ from integration.requirements.requirements import * -from integration.tests.steps.sql import * from integration.tests.steps.service_configurations import * +from integration.tests.steps.sql import * from integration.tests.steps.statements import * diff --git a/sink-connector/tests/integration/tests/columns_inconsistency.py b/sink-connector/tests/integration/tests/columns_inconsistency.py index 0b915e2a1..3f0b9c3c5 100644 --- a/sink-connector/tests/integration/tests/columns_inconsistency.py +++ b/sink-connector/tests/integration/tests/columns_inconsistency.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/consistency.py b/sink-connector/tests/integration/tests/consistency.py index e034a39f3..ecbf65f58 100644 --- a/sink-connector/tests/integration/tests/consistency.py +++ b/sink-connector/tests/integration/tests/consistency.py @@ -1,9 +1,10 @@ -import time from itertools import combinations + from testflows.connect import Shell -from integration.tests.steps.sql import * -from integration.tests.steps.service_configurations import * + from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * +from integration.tests.steps.sql import * @TestOutline diff --git a/sink-connector/tests/integration/tests/deduplication.py b/sink-connector/tests/integration/tests/deduplication.py index f4e9673cb..d47a67467 100644 --- a/sink-connector/tests/integration/tests/deduplication.py +++ b/sink-connector/tests/integration/tests/deduplication.py @@ -1,6 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * @TestOutline diff --git a/sink-connector/tests/integration/tests/delete.py b/sink-connector/tests/integration/tests/delete.py index 08b78b289..1db78df69 100644 --- a/sink-connector/tests/integration/tests/delete.py +++ b/sink-connector/tests/integration/tests/delete.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/insert.py b/sink-connector/tests/integration/tests/insert.py index 76c5a9331..e6a7803c9 100644 --- a/sink-connector/tests/integration/tests/insert.py +++ b/sink-connector/tests/integration/tests/insert.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/multiple_tables.py b/sink-connector/tests/integration/tests/multiple_tables.py index f37ddd246..d6b4897e7 100644 --- a/sink-connector/tests/integration/tests/multiple_tables.py +++ b/sink-connector/tests/integration/tests/multiple_tables.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/partition_limits.py b/sink-connector/tests/integration/tests/partition_limits.py index 7143cf96c..dc1b4f3ea 100644 --- a/sink-connector/tests/integration/tests/partition_limits.py +++ b/sink-connector/tests/integration/tests/partition_limits.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/primary_keys.py b/sink-connector/tests/integration/tests/primary_keys.py index 1f2806d7b..b3af108ba 100644 --- a/sink-connector/tests/integration/tests/primary_keys.py +++ b/sink-connector/tests/integration/tests/primary_keys.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index f3270418a..6f9600e7a 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -1,16 +1,13 @@ -from testflows.core import * -from integration.tests.steps.mysql.mysql import * from integration.tests.steps.clickhouse import * +from integration.tests.steps.mysql.mysql import * from integration.tests.steps.service_configurations import ( init_sink_connector, init_debezium_connector, ) from integration.tests.steps.sql import generate_interesting_table_names from integration.tests.steps.statements import ( - all_ch_datatypes_dict, all_mysql_datatypes_dict, ) -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/sanity.py b/sink-connector/tests/integration/tests/sanity.py index 5cd18a08e..d09dc5695 100644 --- a/sink-connector/tests/integration/tests/sanity.py +++ b/sink-connector/tests/integration/tests/sanity.py @@ -1,7 +1,6 @@ +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/schema_changes.py b/sink-connector/tests/integration/tests/schema_changes.py index 4fb9969ae..509c428e1 100644 --- a/sink-connector/tests/integration/tests/schema_changes.py +++ b/sink-connector/tests/integration/tests/schema_changes.py @@ -1,7 +1,6 @@ +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/steps/statements.py b/sink-connector/tests/integration/tests/steps/statements.py index 132c0c189..632751b5e 100644 --- a/sink-connector/tests/integration/tests/steps/statements.py +++ b/sink-connector/tests/integration/tests/steps/statements.py @@ -187,6 +187,7 @@ "x_varbinary": "String", } +# FIXME datatypes_to_add = ( "TINYBLOB, TINYTEXT, SET, BIT, BOOL, BOOLEAN,INTEGER, FLOAT(size, d), FLOAT(p), DOUBLE PRECISION(" "size, d), DEC, TIMESTAMP, YEAR, ENUM(val1, val2, val3, ...)" diff --git a/sink-connector/tests/integration/tests/sysbench.py b/sink-connector/tests/integration/tests/sysbench.py index 700637882..433cafaec 100644 --- a/sink-connector/tests/integration/tests/sysbench.py +++ b/sink-connector/tests/integration/tests/sysbench.py @@ -1,7 +1,6 @@ from datetime import datetime -from integration.tests.steps.sql import * + from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestScenario diff --git a/sink-connector/tests/integration/tests/truncate.py b/sink-connector/tests/integration/tests/truncate.py index 6768b2e89..da7831e36 100644 --- a/sink-connector/tests/integration/tests/truncate.py +++ b/sink-connector/tests/integration/tests/truncate.py @@ -1,7 +1,6 @@ +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/types.py b/sink-connector/tests/integration/tests/types.py index fa56f3c72..a6c47d2fa 100644 --- a/sink-connector/tests/integration/tests/types.py +++ b/sink-connector/tests/integration/tests/types.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/update.py b/sink-connector/tests/integration/tests/update.py index 459d9e52b..845078f75 100644 --- a/sink-connector/tests/integration/tests/update.py +++ b/sink-connector/tests/integration/tests/update.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline diff --git a/sink-connector/tests/integration/tests/virtual_columns.py b/sink-connector/tests/integration/tests/virtual_columns.py index 919cd3725..1d69c5a66 100644 --- a/sink-connector/tests/integration/tests/virtual_columns.py +++ b/sink-connector/tests/integration/tests/virtual_columns.py @@ -1,7 +1,7 @@ +from integration.requirements.requirements import * +from integration.tests.steps.service_configurations import * from integration.tests.steps.sql import * from integration.tests.steps.statements import * -from integration.tests.steps.service_configurations import * -from integration.requirements.requirements import * @TestOutline From 24d9ea8a1b7f954c226c00b22c7fffd61a4caa26 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 2 Aug 2024 21:51:07 +0400 Subject: [PATCH 15/44] update imports --- .../tests/integration/tests/replication.py | 44 ++++++++++------- .../tests/steps/service_configurations.py | 48 +++++++++++++++++++ .../tests/integration/tests/steps/sql.py | 3 ++ 3 files changed, 78 insertions(+), 17 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index 6f9600e7a..ad9bab4bc 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -10,17 +10,17 @@ ) -@TestOutline +@TestCheck def auto_create_table( self, column_datatype, column_name, - databases, node=None, table_name=None, replicate=False, ): """Check that tables created on the source database are replicated on the destination.""" + databases = self.context.databases if table_name is None: table_name = "table_" + getuid() @@ -49,6 +49,8 @@ def auto_create_table( values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", ) + # pause() + with Then("I check that the table is replicated on the destination database"): with Check(f"table with {column_datatype} was replicated"): check_if_table_was_created( @@ -57,65 +59,73 @@ def auto_create_table( @TestScenario -def check_auto_creation_all_datatypes(self, databases=None, table_name=None): +def check_auto_creation_all_datatypes(self, table_name=None): """Check that tables created on the source database are replicated on the destination.""" - - if databases is None: - databases = ["test"] - for name, datatype in all_mysql_datatypes_dict.items(): ( - Check(test=auto_create_table, name=f"auto table creation {datatype}")( + Check( + test=auto_create_table, + name=f"auto table creation with {datatype} datatype", + )( column_name=name, column_datatype=datatype, - databases=databases, table_name=table_name, ) ) @TestScenario -def auto_creation_different_table_names(self, databases=None): +def auto_creation_different_table_names(self): """Check that tables with all datatypes are replicated on the destination table when tables have names with special cases.""" table_names = generate_interesting_table_names(self.context.number_of_tables) for table_name in table_names: - Check(test=check_auto_creation_all_datatypes)( - databases=databases, table_name=table_name + Check( + test=auto_create_table, + name=f"auto table creation with {table_name} table name", + )( + table_name=table_name, + column_name="name", + column_datatype="VARCHAR(255)", ) @TestScenario def alters(self): """Check that alter statements performed on the source are replicated to the destination.""" - note(1) + pass @TestScenario def inserts(self): """Check that inserts are replicated to the destination.""" - note(1) + pass @TestScenario def deletes(self): """Check that deletes are replicated to the destination.""" - note(1) + pass @TestScenario def updates(self): """Check that updates are replicated to the destination.""" - note(1) + pass @TestFeature @Name("replication") -def feature(self, number_of_tables=150): +def feature(self, number_of_tables=20, databases: list = None): """Check that actions performed on the source database are replicated on the destination database.""" self.context.number_of_tables = number_of_tables + if databases is None: + self.context.databases = ["test"] + else: + self.context.databases = databases + with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() diff --git a/sink-connector/tests/integration/tests/steps/service_configurations.py b/sink-connector/tests/integration/tests/steps/service_configurations.py index 3561e541a..6102314d1 100644 --- a/sink-connector/tests/integration/tests/steps/service_configurations.py +++ b/sink-connector/tests/integration/tests/steps/service_configurations.py @@ -92,6 +92,54 @@ def init_sink_connector( ) +@TestStep(Given) +def init_sink_connector_auto_created(self, topics, node=None, update=None): + """Initialize sink connector with auto created tables.""" + init_sink_connector( + auto_create_tables=True, + topics=topics, + auto_create_replicated_tables=False, + node=node, + update=update, + ) + + +@TestStep(Given) +def init_sink_connector_manual_created(self, topics, node=None, update=None): + """Initialize sink connector with manual created tables.""" + init_sink_connector( + auto_create_tables=False, + topics=topics, + auto_create_replicated_tables=False, + node=node, + update=update, + ) + + +@TestStep(Given) +def init_sink_connector_auto_created_replicated(self, topics, node=None, update=None): + """Initialize sink connector with auto created replicated tables.""" + init_sink_connector( + auto_create_tables=True, + topics=topics, + auto_create_replicated_tables=True, + node=node, + update=update, + ) + + +@TestStep(Given) +def init_sink_connector_manual_created_replicated(self, topics, node=None, update=None): + """Initialize sink connector with manual created replicated tables.""" + init_sink_connector( + auto_create_tables=False, + topics=topics, + auto_create_replicated_tables=True, + node=node, + update=update, + ) + + @TestStep(Given) def init_debezium_connector(self, node=None): """ diff --git a/sink-connector/tests/integration/tests/steps/sql.py b/sink-connector/tests/integration/tests/steps/sql.py index d7c6b2d96..1113b30cc 100644 --- a/sink-connector/tests/integration/tests/steps/sql.py +++ b/sink-connector/tests/integration/tests/steps/sql.py @@ -18,6 +18,8 @@ def generate_interesting_table_names(num_names, max_length=64): special_chars = "_$" utf8_chars = "áéíóúñüç" + chinese_characters = "中文女" + punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_{|}~""" def generate_table_name(length): @@ -27,6 +29,7 @@ def generate_table_name(length): + string.digits + special_chars + utf8_chars + + chinese_characters + punctuation ) return "".join(random.choice(allowed_chars) for _ in range(length)) From 8e9ffb5c22fb1e13b6bd3c8b7ea44f5994d1e853 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 15:17:12 +0400 Subject: [PATCH 16/44] remove pause() --- sink-connector/tests/integration/tests/replication.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index ad9bab4bc..10ced29df 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -13,8 +13,8 @@ @TestCheck def auto_create_table( self, - column_datatype, - column_name, + column_datatype="VARCHAR(255)", + column_name="name", node=None, table_name=None, replicate=False, @@ -49,8 +49,6 @@ def auto_create_table( values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", ) - # pause() - with Then("I check that the table is replicated on the destination database"): with Check(f"table with {column_datatype} was replicated"): check_if_table_was_created( @@ -85,8 +83,6 @@ def auto_creation_different_table_names(self): name=f"auto table creation with {table_name} table name", )( table_name=table_name, - column_name="name", - column_datatype="VARCHAR(255)", ) From 93f8181742adf299546c190ad293d03c8f5b57b1 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 16:17:08 +0400 Subject: [PATCH 17/44] update replication feature --- .../tests/integration/tests/replication.py | 152 ++++++++++++++++-- .../integration/tests/steps/clickhouse.py | 20 +++ .../integration/tests/steps/statements.py | 28 ++++ 3 files changed, 188 insertions(+), 12 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index 10ced29df..14b142983 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -1,4 +1,12 @@ from integration.tests.steps.clickhouse import * +from integration.tests.steps.mysql.alters import ( + add_column, + change_column, + modify_column, + drop_column, + drop_primary_key, + add_primary_key, +) from integration.tests.steps.mysql.mysql import * from integration.tests.steps.service_configurations import ( init_sink_connector, @@ -18,10 +26,14 @@ def auto_create_table( node=None, table_name=None, replicate=False, + validate_values=True, ): """Check that tables created on the source database are replicated on the destination.""" databases = self.context.databases + if type(databases) is not list: + databases = [databases] + if table_name is None: table_name = "table_" + getuid() @@ -44,16 +56,24 @@ def auto_create_table( ) with When("I insert values into the table"): - insert( - table_name=table_name, - values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", - ) + table_values = f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}" + insert(table_name=table_name, values=table_values) + + if not validate_values: + table_values = "" with Then("I check that the table is replicated on the destination database"): with Check(f"table with {column_datatype} was replicated"): check_if_table_was_created( database_name=database, table_name=table_name ) + if validate_values: + validate_data_in_clickhouse_table( + table_name=table_name, + expected_output=table_values.replace("'", ""), + database_name=database, + statement=f"id, {column_name}", + ) @TestScenario @@ -87,15 +107,103 @@ def auto_creation_different_table_names(self): @TestScenario -def alters(self): - """Check that alter statements performed on the source are replicated to the destination.""" - pass +def add_column_on_source(self, database): + """Check that the column is added on the table when we add a column on a database.""" + table_name = f"table_{getuid()}" + column = "new_col" + + with Given("I create a table on multiple databases"): + auto_create_table(table_name=table_name, databases=database) + + with When("I add a column on the table"): + add_column(table_name=table_name, column_name=column, database=database) + + with Then("I check that the column was added on the table"): + check_column(table_name=table_name, column_name=column, database=database) @TestScenario -def inserts(self): - """Check that inserts are replicated to the destination.""" - pass +def change_column_on_source(self, database): + """Check that the column is changed on the table when we change a column on a database.""" + table_name = f"table_{getuid()}" + column = "col1" + new_column = "new_col" + new_column_type = "varchar(255)" + + with Given("I create a table on multiple databases"): + auto_create_table(table_name=table_name, databases=database) + + with When("I change a column on the table"): + change_column( + table_name=table_name, + database=database, + column_name=column, + new_column_name=new_column, + new_column_type=new_column_type, + ) + + with Then("I check that the column was changed on the table"): + check_column(table_name=table_name, column_name=new_column, database=database) + + +@TestScenario +def modify_column_on_source(self, database): + """Check that the column is modified on the table when we modify a column on a database.""" + table_name = f"table_{getuid()}" + column = "col1" + new_column_type = "varchar(255)" + + with Given("I create a table on multiple databases"): + auto_create_table(table_name=table_name, databases=database) + + with When("I modify a column on the table"): + modify_column( + table_name=table_name, + database=database, + column_name=column, + new_column_type=new_column_type, + ) + + with Then("I check that the column was modified on the table"): + check_column( + table_name=table_name, + database=database, + column_name=column, + column_type=new_column_type, + ) + + +@TestScenario +def drop_column_on_source(self, database): + """Check that the column is dropped from the table when we drop a column on a database.""" + table_name = f"table_{getuid()}" + column = "col1" + + with Given("I create a table on multiple databases"): + auto_create_table(table_name=table_name, databases=database) + + with When("I drop a column on the table"): + drop_column(table_name=table_name, database=database, column_name=column) + + with Then("I check that the column was dropped from the table"): + check_column(table_name=table_name, database=database, column_name="") + + +@TestScenario +def add_primary_key_on_a_database(self, database): + """Check that the primary key is added to the table when we add a primary key on a database.""" + table_name = f"table_{getuid()}" + column = "col1" + + with Given("I create a table on multiple databases"): + auto_create_table(table_name=table_name, databases=database) + + with When("I add a primary key on the table"): + drop_primary_key(table_name=table_name, database=database) + add_primary_key(table_name=table_name, database=database, column_name=column) + + with Then("I check that the primary key was added to the table"): + check_column(table_name=table_name, database=database, column_name=column) @TestScenario @@ -110,6 +218,26 @@ def updates(self): pass +@TestSuite +def table_creation(self): + """Check that tables created on the source database are correctly replicated on the destination.""" + Scenario(run=check_auto_creation_all_datatypes) + Scenario(run=auto_creation_different_table_names) + + +@TestSuite +def alters(self): + """Check that alter statements performed on the source are replicated to the destination.""" + databases = self.context.databases + + for database in databases: + Scenario(run=add_column_on_source, database=database) + Scenario(run=change_column_on_source, database=database) + Scenario(run=modify_column_on_source, database=database) + Scenario(run=drop_column_on_source, database=database) + Scenario(run=add_primary_key_on_a_database, database=database) + + @TestFeature @Name("replication") def feature(self, number_of_tables=20, databases: list = None): @@ -125,5 +253,5 @@ def feature(self, number_of_tables=20, databases: list = None): with Given("I enable debezium and sink connectors after kafka starts up"): init_debezium_connector() - for scenario in loads(current_module(), Scenario): - scenario() + for suite in loads(current_module(), Suite): + suite() diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index 38b2a3eea..9bf9b0ce3 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -1,4 +1,5 @@ from integration.helpers.common import * +from integration.tests.steps.statements import mysql_to_clickhouse_datatypes_mapping @TestStep(Then) @@ -17,6 +18,25 @@ def drop_database(self, database_name=None, node=None, cluster=None): node.query(rf"DROP DATABASE IF EXISTS {database_name} ON CLUSTER {cluster};") +@TestStep +def select_column_type(self, node, database, table_name, column_name): + """Check column type in ClickHouse table.""" + with By(f"checking column type for {column_name}"): + node.query( + f"SELECT type FROM system.columns WHERE table = '{table_name}' AND name = '{column_name}' AND database = '{database}' FORMAT TabSeparated" + ).output.strip() + + +@TestStep +def validate_column_type(self, mysq_column_type): + """Validate column type in ClickHouse table.""" + with By(f"validating column type"): + assert ( + mysql_to_clickhouse_datatypes_mapping[mysq_column_type]["mysql"] + == mysql_to_clickhouse_datatypes_mapping[mysq_column_type]["clickhouse"] + ), f"expected {mysql_to_clickhouse_datatypes_mapping[mysq_column_type]['mysql']} but got {mysql_to_clickhouse_datatypes_mapping[mysq_column_type]['clickhouse']}" + + @TestStep(Then) def check_column( self, table_name, column_name, node=None, column_type=None, database=None diff --git a/sink-connector/tests/integration/tests/steps/statements.py b/sink-connector/tests/integration/tests/steps/statements.py index 632751b5e..079ee58b0 100644 --- a/sink-connector/tests/integration/tests/steps/statements.py +++ b/sink-connector/tests/integration/tests/steps/statements.py @@ -111,6 +111,34 @@ f" x_varbinary String" ) +mysql_to_clickhouse_datatypes_mapping = { + "DECIMAL(2,1)": {"clickhouse": "Decimal(2,1)", "mysql": "DECIMAL(2,1)"}, + "DECIMAL(30, 10)": {"clickhouse": "Decimal(30,10)", "mysql": "DECIMAL(30, 10)"}, + "DOUBLE": {"clickhouse": "Float64", "mysql": "DOUBLE"}, + "DATE": {"clickhouse": "Date", "mysql": "DATE"}, + "DATETIME(6)": {"clickhouse": "String", "mysql": "DATETIME(6)"}, + "TIME": {"clickhouse": "String", "mysql": "TIME"}, + "TIME(6)": {"clickhouse": "String", "mysql": "TIME(6)"}, + "INT": {"clickhouse": "Int32", "mysql": "INT"}, + "INT UNSIGNED": {"clickhouse": "UInt32", "mysql": "INT UNSIGNED"}, + "BIGINT": {"clickhouse": "UInt64", "mysql": "BIGINT"}, + "BIGINT UNSIGNED": {"clickhouse": "UInt64", "mysql": "BIGINT UNSIGNED"}, + "TINYINT": {"clickhouse": "Int8", "mysql": "TINYINT"}, + "TINYINT UNSIGNED": {"clickhouse": "UInt8", "mysql": "TINYINT UNSIGNED"}, + "SMALLINT": {"clickhouse": "Int16", "mysql": "SMALLINT"}, + "SMALLINT UNSIGNED": {"clickhouse": "UInt16", "mysql": "SMALLINT UNSIGNED"}, + "MEDIUMINT": {"clickhouse": "Int32", "mysql": "MEDIUMINT"}, + "MEDIUMINT UNSIGNED": {"clickhouse": "UInt32", "mysql": "MEDIUMINT UNSIGNED"}, + "CHAR": {"clickhouse": "LowCardinality(String)", "mysql": "CHAR"}, + "TEXT": {"clickhouse": "String", "mysql": "TEXT"}, + "VARCHAR(4)": {"clickhouse": "String", "mysql": "VARCHAR(4)"}, + "BLOB": {"clickhouse": "String", "mysql": "BLOB"}, + "MEDIUMBLOB": {"clickhouse": "String", "mysql": "MEDIUMBLOB"}, + "LONGBLOB": {"clickhouse": "String", "mysql": "LONGBLOB"}, + "BINARY": {"clickhouse": "String", "mysql": "BINARY"}, + "VARBINARY(4)": {"clickhouse": "String", "mysql": "VARBINARY(4)"}, +} + all_mysql_datatypes_dict = { "D4": "DECIMAL(2,1) NOT NULL", "D5": "DECIMAL(30, 10) NOT NULL", From 3da9c9ca512c6812d7b5d5c1466c861c6f0d7bab Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 17:25:45 +0400 Subject: [PATCH 18/44] update replication with deletes --- .../tests/integration/tests/replication.py | 68 +++++++++++++++++-- .../integration/tests/steps/mysql/deletes.py | 3 +- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index 14b142983..ed3b21638 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -7,6 +7,7 @@ drop_primary_key, add_primary_key, ) +from integration.tests.steps.mysql.deletes import delete_all_records, delete from integration.tests.steps.mysql.mysql import * from integration.tests.steps.service_configurations import ( init_sink_connector, @@ -18,7 +19,7 @@ ) -@TestCheck +@TestOutline def auto_create_table( self, column_datatype="VARCHAR(255)", @@ -27,6 +28,7 @@ def auto_create_table( table_name=None, replicate=False, validate_values=True, + multiple_inserts=False, ): """Check that tables created on the source database are replicated on the destination.""" databases = self.context.databases @@ -56,9 +58,15 @@ def auto_create_table( ) with When("I insert values into the table"): - table_values = f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}" - insert(table_name=table_name, values=table_values) - + if not multiple_inserts: + table_values = f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}" + insert(table_name=table_name, values=table_values) + else: + for _ in range(10): + insert( + table_name=table_name, + values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", + ) if not validate_values: table_values = "" @@ -76,6 +84,16 @@ def auto_create_table( ) +@TestStep(Given) +def auto_create_with_multiple_inserts(self, table_name=None): + """Create a table with multiple inserts.""" + auto_create_table( + table_name=table_name, + multiple_inserts=True, + validate_values=False, + ) + + @TestScenario def check_auto_creation_all_datatypes(self, table_name=None): """Check that tables created on the source database are replicated on the destination.""" @@ -207,9 +225,35 @@ def add_primary_key_on_a_database(self, database): @TestScenario -def deletes(self): - """Check that deletes are replicated to the destination.""" - pass +def delete_all_records_from_source(self): + """Check that records are deleted from the destination table when we delete all columns on the source.""" + table_name = f"table_{getuid()}" + + with Given("I create a table on multiple databases"): + auto_create_with_multiple_inserts(table_name=table_name) + + for database in self.context.databases: + with When("I delete columns on the source"): + delete_all_records(table_name=table_name, database=database) + + with Then("I check that the primary key was added to the table"): + select(table_name=table_name, database=database, manual_output="") + + +@TestScenario +def delete_specific_records(self): + """Check that records are deleted from the destination table when we execute DELETE WHERE on source table.""" + table_name = f"table_{getuid()}" + + with Given("I create a table on multiple databases"): + auto_create_with_multiple_inserts(table_name=table_name) + + for database in self.context.databases: + with When("I delete columns on the source"): + delete(table_name=table_name, database=database, condition="WHERE id > 0") + + with Then("I check that the primary key was added to the table"): + select(table_name=table_name, database=database, manual_output="") @TestScenario @@ -238,6 +282,16 @@ def alters(self): Scenario(run=add_primary_key_on_a_database, database=database) +@TestSuite +def deletes(self): + """Check that deletes are replicated to the destination.""" + databases = self.context.databases + + for database in databases: + Scenario(run=delete_all_records_from_source, database=database) + Scenario(run=delete_specific_records, database=database) + + @TestFeature @Name("replication") def feature(self, number_of_tables=20, databases: list = None): diff --git a/sink-connector/tests/integration/tests/steps/mysql/deletes.py b/sink-connector/tests/integration/tests/steps/mysql/deletes.py index 47ebf508e..badfeea29 100644 --- a/sink-connector/tests/integration/tests/steps/mysql/deletes.py +++ b/sink-connector/tests/integration/tests/steps/mysql/deletes.py @@ -7,7 +7,7 @@ def delete(self, table, condition=None, database=None, node=None): if node is None: node = self.context.cluster.node("mysql-master") - query = rf"DELETE FROM {database}.{table} WHERE {condition};" + query = rf"DELETE FROM {database}.{table}{condition};" if condition is not None: query += f" WHERE {condition}" @@ -24,4 +24,3 @@ def delete_all_records(self, table_name, database=None, node=None): with By("executing DELETE query"): delete(node=node, table=table_name, database=database, condition=None) - node.query(rf"DELETE FROM {database}.{table_name};") From cf81d99e2a0dc811acbecf00b18b700fc6487e11 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 17:34:02 +0400 Subject: [PATCH 19/44] optimize imports --- .../tests/integration/tests/replication.py | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index ed3b21638..938610336 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -1,4 +1,12 @@ -from integration.tests.steps.clickhouse import * +from testflows.core import * + +from integration.helpers.common import getuid +from integration.tests.steps.clickhouse import ( + check_if_table_was_created, + validate_data_in_clickhouse_table, + check_column, + select, +) from integration.tests.steps.mysql.alters import ( add_column, change_column, @@ -8,7 +16,12 @@ add_primary_key, ) from integration.tests.steps.mysql.deletes import delete_all_records, delete -from integration.tests.steps.mysql.mysql import * +from integration.tests.steps.mysql.mysql import ( + create_mysql_table, + insert, + generate_sample_mysql_value, +) +from integration.tests.steps.mysql.updates import update from integration.tests.steps.service_configurations import ( init_sink_connector, init_debezium_connector, @@ -257,9 +270,19 @@ def delete_specific_records(self): @TestScenario -def updates(self): - """Check that updates are replicated to the destination.""" - pass +def update_record_on_source(self): + """Check that the record is updated on the destination table when we update a record on the source.""" + table_name = f"table_{getuid()}" + column = "col1" + + with Given("I create a table on multiple databases"): + auto_create_table(table_name=table_name) + + with When("I update a record on the source"): + update(table_name=table_name, column_name=column) + + with Then("I check that the record was updated on the destination"): + select(table_name=table_name, manual_output="") @TestSuite From 0a3490ae01b1d41819f86aa7ea9f299ad59914e5 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 17:58:12 +0400 Subject: [PATCH 20/44] update replication with updates --- .../tests/integration/tests/replication.py | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index 938610336..ac54ad1d1 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -6,6 +6,7 @@ validate_data_in_clickhouse_table, check_column, select, + get_random_value_from_column, ) from integration.tests.steps.mysql.alters import ( add_column, @@ -273,16 +274,30 @@ def delete_specific_records(self): def update_record_on_source(self): """Check that the record is updated on the destination table when we update a record on the source.""" table_name = f"table_{getuid()}" - column = "col1" with Given("I create a table on multiple databases"): - auto_create_table(table_name=table_name) + auto_create_with_multiple_inserts(table_name=table_name) + + for database in self.context.databases: + with When("I update a record on the source"): + random_value = get_random_value_from_column( + database=database, table_name=table_name, column_name="id" + ) - with When("I update a record on the source"): - update(table_name=table_name, column_name=column) + update( + table_name=table_name, + database=database, + set="id 5", + condition=f"id = {random_value}", + ) - with Then("I check that the record was updated on the destination"): - select(table_name=table_name, manual_output="") + with Then("I check that the primary key was added to the table"): + select( + table_name=table_name, + database=database, + where=f"id = {random_value}", + manual_output=random_value, + ) @TestSuite @@ -314,6 +329,11 @@ def deletes(self): Scenario(run=delete_all_records_from_source, database=database) Scenario(run=delete_specific_records, database=database) +@TestSuite +def updates(self): + """Check that updates are replicated to the destination.""" + Scenario(run=update_record_on_source) + @TestFeature @Name("replication") From 9093c176d6f198dd39c22ec04dfbeeec1ed839de Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 17:58:40 +0400 Subject: [PATCH 21/44] add where clause to the clickhouse --- .../integration/tests/steps/clickhouse.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index 9bf9b0ce3..4bd5686e9 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -110,6 +110,7 @@ def select( with_optimize=False, sign_column="_sign", timeout=300, + where=None, ): """SELECT statement in ClickHouse with an option to use FINAL or loop SELECT + OPTIMIZE TABLE default simple 'SELECT'""" if node is None: @@ -148,12 +149,19 @@ def select( message=f"{manual_output}", ) else: + r = "SELECT {statement} FROM {database}.{table_name}" + + if where: + r += f" WHERE {where}" + + r += " FORMAT CSV" + retry( node.query, timeout=timeout, delay=10, )( - f"SELECT {statement} FROM {database}.{table_name} FORMAT CSV", + r, message=f"{manual_output}", ) @@ -316,3 +324,22 @@ def verify_table_creation_in_clickhouse( with_optimize=with_optimize, timeout=timeout, ) + + +@TestStep(When) +def get_random_value_from_column( + self, table_name, column_name, database=None, node=None +): + """Get random value from ClickHouse table column.""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("clickhouse") + + with By(f"getting random value from {column_name}"): + random_value = node.query( + rf"SELECT {column_name} FROM {database}.\`{table_name}\` ORDER BY rand() LIMIT 1" + ) + + return random_value From 725cc75b1acd1b70afa821e2b79b241c410bc6d6 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 18:13:12 +0400 Subject: [PATCH 22/44] correct imports --- sink-connector/tests/integration/tests/autocreate.py | 4 ++-- .../tests/integration/tests/columns_inconsistency.py | 4 ++-- sink-connector/tests/integration/tests/consistency.py | 2 +- sink-connector/tests/integration/tests/deduplication.py | 4 ++-- sink-connector/tests/integration/tests/delete.py | 4 ++-- sink-connector/tests/integration/tests/insert.py | 4 ++-- sink-connector/tests/integration/tests/multiple_tables.py | 4 ++-- sink-connector/tests/integration/tests/partition_limits.py | 4 ++-- sink-connector/tests/integration/tests/primary_keys.py | 4 ++-- sink-connector/tests/integration/tests/replication.py | 6 +++--- sink-connector/tests/integration/tests/sanity.py | 4 ++-- sink-connector/tests/integration/tests/schema_changes.py | 4 ++-- sink-connector/tests/integration/tests/sysbench.py | 2 +- sink-connector/tests/integration/tests/truncate.py | 4 ++-- sink-connector/tests/integration/tests/types.py | 4 ++-- sink-connector/tests/integration/tests/update.py | 4 ++-- sink-connector/tests/integration/tests/virtual_columns.py | 4 ++-- 17 files changed, 33 insertions(+), 33 deletions(-) diff --git a/sink-connector/tests/integration/tests/autocreate.py b/sink-connector/tests/integration/tests/autocreate.py index 4698c2af4..a64757588 100644 --- a/sink-connector/tests/integration/tests/autocreate.py +++ b/sink-connector/tests/integration/tests/autocreate.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/columns_inconsistency.py b/sink-connector/tests/integration/tests/columns_inconsistency.py index 3f0b9c3c5..20e256b6d 100644 --- a/sink-connector/tests/integration/tests/columns_inconsistency.py +++ b/sink-connector/tests/integration/tests/columns_inconsistency.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/consistency.py b/sink-connector/tests/integration/tests/consistency.py index ecbf65f58..10aeaf86d 100644 --- a/sink-connector/tests/integration/tests/consistency.py +++ b/sink-connector/tests/integration/tests/consistency.py @@ -3,7 +3,7 @@ from testflows.connect import Shell from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * diff --git a/sink-connector/tests/integration/tests/deduplication.py b/sink-connector/tests/integration/tests/deduplication.py index d47a67467..38f862f1e 100644 --- a/sink-connector/tests/integration/tests/deduplication.py +++ b/sink-connector/tests/integration/tests/deduplication.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/delete.py b/sink-connector/tests/integration/tests/delete.py index 1db78df69..b5a12ae83 100644 --- a/sink-connector/tests/integration/tests/delete.py +++ b/sink-connector/tests/integration/tests/delete.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/insert.py b/sink-connector/tests/integration/tests/insert.py index e6a7803c9..ddccb47ad 100644 --- a/sink-connector/tests/integration/tests/insert.py +++ b/sink-connector/tests/integration/tests/insert.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/multiple_tables.py b/sink-connector/tests/integration/tests/multiple_tables.py index d6b4897e7..1e8c4fff3 100644 --- a/sink-connector/tests/integration/tests/multiple_tables.py +++ b/sink-connector/tests/integration/tests/multiple_tables.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/partition_limits.py b/sink-connector/tests/integration/tests/partition_limits.py index dc1b4f3ea..ec375c623 100644 --- a/sink-connector/tests/integration/tests/partition_limits.py +++ b/sink-connector/tests/integration/tests/partition_limits.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/primary_keys.py b/sink-connector/tests/integration/tests/primary_keys.py index b3af108ba..788744a05 100644 --- a/sink-connector/tests/integration/tests/primary_keys.py +++ b/sink-connector/tests/integration/tests/primary_keys.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index ac54ad1d1..af7fdc6a7 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -23,12 +23,12 @@ generate_sample_mysql_value, ) from integration.tests.steps.mysql.updates import update -from integration.tests.steps.service_configurations import ( +from integration.tests.steps.configurations import ( init_sink_connector, init_debezium_connector, ) from integration.tests.steps.sql import generate_interesting_table_names -from integration.tests.steps.statements import ( +from integration.tests.steps.datatypes import ( all_mysql_datatypes_dict, ) @@ -337,7 +337,7 @@ def updates(self): @TestFeature @Name("replication") -def feature(self, number_of_tables=20, databases: list = None): +def replication(self, number_of_tables=20, databases: list = None): """Check that actions performed on the source database are replicated on the destination database.""" self.context.number_of_tables = number_of_tables diff --git a/sink-connector/tests/integration/tests/sanity.py b/sink-connector/tests/integration/tests/sanity.py index d09dc5695..b59eb2606 100644 --- a/sink-connector/tests/integration/tests/sanity.py +++ b/sink-connector/tests/integration/tests/sanity.py @@ -1,6 +1,6 @@ -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/schema_changes.py b/sink-connector/tests/integration/tests/schema_changes.py index 509c428e1..747edbd4b 100644 --- a/sink-connector/tests/integration/tests/schema_changes.py +++ b/sink-connector/tests/integration/tests/schema_changes.py @@ -1,6 +1,6 @@ -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/sysbench.py b/sink-connector/tests/integration/tests/sysbench.py index 433cafaec..b321e274b 100644 --- a/sink-connector/tests/integration/tests/sysbench.py +++ b/sink-connector/tests/integration/tests/sysbench.py @@ -1,6 +1,6 @@ from datetime import datetime -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * @TestScenario diff --git a/sink-connector/tests/integration/tests/truncate.py b/sink-connector/tests/integration/tests/truncate.py index da7831e36..694597d7e 100644 --- a/sink-connector/tests/integration/tests/truncate.py +++ b/sink-connector/tests/integration/tests/truncate.py @@ -1,6 +1,6 @@ -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/types.py b/sink-connector/tests/integration/tests/types.py index a6c47d2fa..a30ae9bf3 100644 --- a/sink-connector/tests/integration/tests/types.py +++ b/sink-connector/tests/integration/tests/types.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/update.py b/sink-connector/tests/integration/tests/update.py index 845078f75..718b821b8 100644 --- a/sink-connector/tests/integration/tests/update.py +++ b/sink-connector/tests/integration/tests/update.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline diff --git a/sink-connector/tests/integration/tests/virtual_columns.py b/sink-connector/tests/integration/tests/virtual_columns.py index 1d69c5a66..75048e20d 100644 --- a/sink-connector/tests/integration/tests/virtual_columns.py +++ b/sink-connector/tests/integration/tests/virtual_columns.py @@ -1,7 +1,7 @@ from integration.requirements.requirements import * -from integration.tests.steps.service_configurations import * +from integration.tests.steps.configurations import * from integration.tests.steps.sql import * -from integration.tests.steps.statements import * +from integration.tests.steps.datatypes import * @TestOutline From c07baff02f64ecd7707f48078358f0de6a2d15e1 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 18:14:23 +0400 Subject: [PATCH 23/44] rename files --- .../tests/steps/{service_configurations.py => configurations.py} | 0 .../tests/integration/tests/steps/{statements.py => datatypes.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename sink-connector/tests/integration/tests/steps/{service_configurations.py => configurations.py} (100%) rename sink-connector/tests/integration/tests/steps/{statements.py => datatypes.py} (100%) diff --git a/sink-connector/tests/integration/tests/steps/service_configurations.py b/sink-connector/tests/integration/tests/steps/configurations.py similarity index 100% rename from sink-connector/tests/integration/tests/steps/service_configurations.py rename to sink-connector/tests/integration/tests/steps/configurations.py diff --git a/sink-connector/tests/integration/tests/steps/statements.py b/sink-connector/tests/integration/tests/steps/datatypes.py similarity index 100% rename from sink-connector/tests/integration/tests/steps/statements.py rename to sink-connector/tests/integration/tests/steps/datatypes.py From 472ad091527ba8497f361abd1917d44e1c5a5c39 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 5 Aug 2024 18:14:30 +0400 Subject: [PATCH 24/44] rename files --- sink-connector/tests/integration/tests/steps/clickhouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index 4bd5686e9..6ae54eb5f 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -1,5 +1,5 @@ from integration.helpers.common import * -from integration.tests.steps.statements import mysql_to_clickhouse_datatypes_mapping +from integration.tests.steps.datatypes import mysql_to_clickhouse_datatypes_mapping @TestStep(Then) From 0da33392bad5c9b538f0edd42637526621704c43 Mon Sep 17 00:00:00 2001 From: selfeer Date: Tue, 6 Aug 2024 17:16:41 +0400 Subject: [PATCH 25/44] add default package on workflow call --- .github/workflows/testflows-sink-connector-kafka.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index 66bd21c85..1e6650f13 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -8,6 +8,10 @@ on: description: "Kafka connector docker image" required: true type: string + package: + description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'" + type: string + default: docker://clickhouse/clickhouse-server:23.3 secrets: DOCKERHUB_USERNAME: required: false From 863b12b3c5ecb30bba0589ff5c0c48da6ed565fe Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 9 Aug 2024 15:19:51 +0400 Subject: [PATCH 26/44] add possibility to update sink config before running the replication suite --- sink-connector/tests/integration/tests/replication.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index af7fdc6a7..91572b845 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -46,6 +46,7 @@ def auto_create_table( ): """Check that tables created on the source database are replicated on the destination.""" databases = self.context.databases + update_sink_config = self.context.update_sink_config if type(databases) is not list: databases = [databases] @@ -61,6 +62,7 @@ def auto_create_table( auto_create_tables=True, topics=f"SERVER5432.{database}.{table_name}", auto_create_replicated_tables=replicate, + update=update_sink_config, ) with And("I create a table on the source database"): @@ -329,6 +331,7 @@ def deletes(self): Scenario(run=delete_all_records_from_source, database=database) Scenario(run=delete_specific_records, database=database) + @TestSuite def updates(self): """Check that updates are replicated to the destination.""" @@ -337,10 +340,11 @@ def updates(self): @TestFeature @Name("replication") -def replication(self, number_of_tables=20, databases: list = None): +def replication(self, number_of_tables=20, databases: list = None, update: dict = None): """Check that actions performed on the source database are replicated on the destination database.""" self.context.number_of_tables = number_of_tables + self.context.update_sink_config = update if databases is None: self.context.databases = ["test"] From 36c5fae49aa590c9b6f2d888bf0c10bc3a32aa50 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 9 Aug 2024 17:43:45 +0400 Subject: [PATCH 27/44] update replication suite, restructure mysql steps --- .../tests/integration/tests/replication.py | 128 ++++++++++-------- .../integration/tests/steps/mysql/mysql.py | 49 +++++++ .../tests/integration/tests/steps/sql.py | 48 ------- 3 files changed, 118 insertions(+), 107 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index af7fdc6a7..a3fbb1fa6 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -8,6 +8,13 @@ select, get_random_value_from_column, ) +from integration.tests.steps.configurations import ( + init_sink_connector, + init_debezium_connector, +) +from integration.tests.steps.datatypes import ( + all_mysql_datatypes_dict, +) from integration.tests.steps.mysql.alters import ( add_column, change_column, @@ -21,16 +28,9 @@ create_mysql_table, insert, generate_sample_mysql_value, + generate_special_case_names, ) from integration.tests.steps.mysql.updates import update -from integration.tests.steps.configurations import ( - init_sink_connector, - init_debezium_connector, -) -from integration.tests.steps.sql import generate_interesting_table_names -from integration.tests.steps.datatypes import ( - all_mysql_datatypes_dict, -) @TestOutline @@ -73,7 +73,7 @@ def auto_create_table( with When("I insert values into the table"): if not multiple_inserts: - table_values = f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}" + table_values = f"{generate_sample_mysql_value('INT')},{generate_sample_mysql_value(column_datatype)}" insert(table_name=table_name, values=table_values) else: for _ in range(10): @@ -127,7 +127,7 @@ def check_auto_creation_all_datatypes(self, table_name=None): @TestScenario def auto_creation_different_table_names(self): """Check that tables with all datatypes are replicated on the destination table when tables have names with special cases.""" - table_names = generate_interesting_table_names(self.context.number_of_tables) + table_names = generate_special_case_names(self.context.number_of_tables) for table_name in table_names: Check( @@ -145,13 +145,14 @@ def add_column_on_source(self, database): column = "new_col" with Given("I create a table on multiple databases"): - auto_create_table(table_name=table_name, databases=database) + auto_create_table(table_name=table_name) - with When("I add a column on the table"): - add_column(table_name=table_name, column_name=column, database=database) + for database in self.context.databases: + with When("I add a column on the table"): + add_column(table_name=table_name, column_name=column, database=database) - with Then("I check that the column was added on the table"): - check_column(table_name=table_name, column_name=column, database=database) + with Then("I check that the column was added on the table"): + check_column(table_name=table_name, column_name=column, database=database) @TestScenario @@ -163,19 +164,22 @@ def change_column_on_source(self, database): new_column_type = "varchar(255)" with Given("I create a table on multiple databases"): - auto_create_table(table_name=table_name, databases=database) + auto_create_table(table_name=table_name) - with When("I change a column on the table"): - change_column( - table_name=table_name, - database=database, - column_name=column, - new_column_name=new_column, - new_column_type=new_column_type, - ) + for database in self.context.databases: + with When("I change a column on the table"): + change_column( + table_name=table_name, + database=database, + column_name=column, + new_column_name=new_column, + new_column_type=new_column_type, + ) - with Then("I check that the column was changed on the table"): - check_column(table_name=table_name, column_name=new_column, database=database) + with Then("I check that the column was changed on the table"): + check_column( + table_name=table_name, column_name=new_column, database=database + ) @TestScenario @@ -186,23 +190,24 @@ def modify_column_on_source(self, database): new_column_type = "varchar(255)" with Given("I create a table on multiple databases"): - auto_create_table(table_name=table_name, databases=database) + auto_create_table(table_name=table_name) - with When("I modify a column on the table"): - modify_column( - table_name=table_name, - database=database, - column_name=column, - new_column_type=new_column_type, - ) + for database in self.context.databases: + with When("I modify a column on the table"): + modify_column( + table_name=table_name, + database=database, + column_name=column, + new_column_type=new_column_type, + ) - with Then("I check that the column was modified on the table"): - check_column( - table_name=table_name, - database=database, - column_name=column, - column_type=new_column_type, - ) + with Then("I check that the column was modified on the table"): + check_column( + table_name=table_name, + database=database, + column_name=column, + column_type=new_column_type, + ) @TestScenario @@ -212,13 +217,14 @@ def drop_column_on_source(self, database): column = "col1" with Given("I create a table on multiple databases"): - auto_create_table(table_name=table_name, databases=database) + auto_create_table(table_name=table_name) - with When("I drop a column on the table"): - drop_column(table_name=table_name, database=database, column_name=column) + for database in self.context.databases: + with When("I drop a column on the table"): + drop_column(table_name=table_name, database=database, column_name=column) - with Then("I check that the column was dropped from the table"): - check_column(table_name=table_name, database=database, column_name="") + with Then("I check that the column was dropped from the table"): + check_column(table_name=table_name, database=database, column_name="") @TestScenario @@ -228,14 +234,17 @@ def add_primary_key_on_a_database(self, database): column = "col1" with Given("I create a table on multiple databases"): - auto_create_table(table_name=table_name, databases=database) + auto_create_table(table_name=table_name) - with When("I add a primary key on the table"): - drop_primary_key(table_name=table_name, database=database) - add_primary_key(table_name=table_name, database=database, column_name=column) + for database in self.context.databases: + with When("I add a primary key on the table"): + drop_primary_key(table_name=table_name, database=database) + add_primary_key( + table_name=table_name, database=database, column_name=column + ) - with Then("I check that the primary key was added to the table"): - check_column(table_name=table_name, database=database, column_name=column) + with Then("I check that the primary key was added to the table"): + check_column(table_name=table_name, database=database, column_name=column) @TestScenario @@ -313,11 +322,11 @@ def alters(self): databases = self.context.databases for database in databases: - Scenario(run=add_column_on_source, database=database) - Scenario(run=change_column_on_source, database=database) - Scenario(run=modify_column_on_source, database=database) - Scenario(run=drop_column_on_source, database=database) - Scenario(run=add_primary_key_on_a_database, database=database) + Scenario(test=add_column_on_source)(database=database) + Scenario(test=change_column_on_source)(database=database) + Scenario(test=modify_column_on_source)(database=database) + Scenario(test=drop_column_on_source)(database=database) + Scenario(test=add_primary_key_on_a_database)(database=database) @TestSuite @@ -326,8 +335,9 @@ def deletes(self): databases = self.context.databases for database in databases: - Scenario(run=delete_all_records_from_source, database=database) - Scenario(run=delete_specific_records, database=database) + Scenario(test=delete_all_records_from_source)(database=database) + Scenario(test=delete_specific_records)(database=database) + @TestSuite def updates(self): diff --git a/sink-connector/tests/integration/tests/steps/mysql/mysql.py b/sink-connector/tests/integration/tests/steps/mysql/mysql.py index 1cee7e4c5..d353726b8 100644 --- a/sink-connector/tests/integration/tests/steps/mysql/mysql.py +++ b/sink-connector/tests/integration/tests/steps/mysql/mysql.py @@ -1,10 +1,59 @@ import random +import string from integration.helpers.common import * from datetime import datetime, timedelta from testflows.core import * +def generate_special_case_names(num_names, max_length=64): + """Generate a list of interesting MySQL table and database names for testing.""" + reserved_keywords = [ + "Select", + "Insert", + "Update", + "Delete", + "Group", + "Where", + "Transaction", + ] + + special_chars = "_$" + utf8_chars = "áéíóúñüç" + chinese_characters = "中文女" + + punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_{|}~""" + + def generate_table_name(length): + """Generate a random table name of a given length.""" + allowed_chars = ( + string.ascii_letters + + string.digits + + special_chars + + utf8_chars + + chinese_characters + + punctuation + ) + return "".join(random.choice(allowed_chars) for _ in range(length)) + + table_names = set() + + # Add reserved keywords + table_names.update(reserved_keywords) + + while len(table_names) < num_names: + length = random.randint(1, max_length) + name = generate_table_name(length) + + # Ensure the name does not start with a digit + if name[0] in string.digits: + name = "_" + name + + table_names.add(f"{name}") + + return list(table_names) + + def generate_sample_mysql_value(data_type): """Generate a sample MySQL value for the provided datatype.""" if data_type.startswith("DECIMAL"): diff --git a/sink-connector/tests/integration/tests/steps/sql.py b/sink-connector/tests/integration/tests/steps/sql.py index 1113b30cc..4d1f5be65 100644 --- a/sink-connector/tests/integration/tests/steps/sql.py +++ b/sink-connector/tests/integration/tests/steps/sql.py @@ -4,54 +4,6 @@ from integration.helpers.common import * -def generate_interesting_table_names(num_names, max_length=64): - """Generate a list of interesting MySQL table names for testing.""" - reserved_keywords = [ - "Select", - "Insert", - "Update", - "Delete", - "Group", - "Where", - "Transaction", - ] - - special_chars = "_$" - utf8_chars = "áéíóúñüç" - chinese_characters = "中文女" - - punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_{|}~""" - - def generate_table_name(length): - """Generate a random table name of a given length.""" - allowed_chars = ( - string.ascii_letters - + string.digits - + special_chars - + utf8_chars - + chinese_characters - + punctuation - ) - return "".join(random.choice(allowed_chars) for _ in range(length)) - - table_names = set() - - # Add reserved keywords - table_names.update(reserved_keywords) - - while len(table_names) < num_names: - length = random.randint(1, max_length) - name = generate_table_name(length) - - # Ensure the name does not start with a digit - if name[0] in string.digits: - name = "_" + name - - table_names.add(f"{name}") - - return list(table_names) - - @TestStep(Given) def create_mysql_table(self, name=None, statement=None, node=None): """ From 61c2ac2d874d3755dad39a9ff07ce856d3d38f2d Mon Sep 17 00:00:00 2001 From: Selfeer Date: Fri, 9 Aug 2024 18:48:49 +0400 Subject: [PATCH 28/44] add multiple databases --- .../tests/integration/regression.py | 7 ++ .../integration/tests/multiple_databases.py | 71 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 sink-connector/tests/integration/tests/multiple_databases.py diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index 9e4a65f4a..41e73720b 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -40,6 +40,11 @@ "types/date time/*": [(Fail, "difference between timezones, tests need rework")], "types/integer types/*": [(Fail, "requires investigation")], } + +ffails = { + "/regression/multiple databases": (Skip, "Work in progress") +} + xflags = {} @@ -47,6 +52,7 @@ @ArgumentParser(argparser) @XFails(xfails) @XFlags(xflags) +@FFails(ffails) @Name("regression") @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication("1.0"), @@ -117,6 +123,7 @@ def regression( Feature(run=load("tests.primary_keys", "feature")) Feature(run=load("tests.virtual_columns", "feature")) Feature(run=load("tests.columns_inconsistency", "feature")) + Feature(run=load("tests.multiple_databases", "feature")) if __name__ == "__main__": diff --git a/sink-connector/tests/integration/tests/multiple_databases.py b/sink-connector/tests/integration/tests/multiple_databases.py new file mode 100644 index 000000000..51a25a324 --- /dev/null +++ b/sink-connector/tests/integration/tests/multiple_databases.py @@ -0,0 +1,71 @@ +from testflows.core import * +from integration.tests.replication import replication +from integration.tests.steps.mysql.mysql import ( + create_mysql_database, + generate_special_case_names, +) +from integration.tests.steps.clickhouse import create_clickhouse_database + + +@TestStep(Given) +def create_source_database(self, database_name): + """Create a MySQL database.""" + create_mysql_database(database_name=database_name) + + +@TestStep(Given) +def create_destination_database(self, database_name): + """Create a ClickHouse database.""" + create_clickhouse_database(name=database_name) + + +@TestStep(Given) +def create_source_and_destination_databases(self, database_name=None): + """Create databases where MySQL database is source and ClickHouse database is destination.""" + if database_name is None: + database_name = "test" + + with By(f"creating a ClickHouse database {database_name}"): + create_clickhouse_database(name=database_name) + + with And(f"creating a a MySQL database {database_name}"): + create_mysql_database(database_name=database_name) + + +@TestScenario +def create_databases_with_special_names(self): + """Create databases with special names on source and destination manually.""" + update_config = { + "auto.create.tables": "true", + "auto.create.tables.replicated": "false", + } + databases = generate_special_case_names(self.context.number_of_databases) + + for database in databases: + create_source_and_destination_databases(database_name=rf"\`{database}\`") + + replication(databases=databases, update=update_config) + + +@TestScenario +def auto_create_databases_with_special_names(self): + """Auto create databases with special characters in the name on source and wait and check it's created on destination.""" + databases = generate_special_case_names(self.context.number_of_databases) + update_config = { + "auto.create.tables": "true", + "auto.create.tables.replicated": "false", + } + for database in databases: + create_source_database(database_name=rf"\`{database}\`") + + replication(databases=databases, update=update_config) + + +@TestFeature +@Name("multiple databases") +def feature(self, number_of_databases=100): + """Validate sink connector with multiple databases.""" + self.context.number_of_databases = number_of_databases + + Scenario(run=create_databases_with_special_names) + Scenario(run=auto_create_databases_with_special_names) From 18674ba5c2f0abca4d4ea23e0a7e94be31699636 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:32:34 +0400 Subject: [PATCH 29/44] add backticks to database names --- .../integration/tests/steps/clickhouse.py | 10 ++-- .../integration/tests/steps/mysql/mysql.py | 57 +++++++++++++++---- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index 6ae54eb5f..08f40c02e 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -15,7 +15,9 @@ def drop_database(self, database_name=None, node=None, cluster=None): if node is None: node = self.context.cluster.node("clickhouse") with By("executing drop database query"): - node.query(rf"DROP DATABASE IF EXISTS {database_name} ON CLUSTER {cluster};") + node.query( + rf"DROP DATABASE IF EXISTS \`{database_name}\` ON CLUSTER {cluster};" + ) @TestStep @@ -90,7 +92,7 @@ def create_clickhouse_database(self, name=None, node=None): drop_database(database_name=name) node.query( - rf"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER replicated_cluster" + rf"CREATE DATABASE IF NOT EXISTS \`{name}\` ON CLUSTER replicated_cluster" ) yield finally: @@ -186,11 +188,11 @@ def check_if_table_was_created( if replicated: for node in self.context.cluster.nodes["clickhouse"]: retry(self.context.cluster.node(node).query, timeout=timeout, delay=3)( - rf"EXISTS {database_name}.\`{table_name}\`", message=f"{message}" + rf"EXISTS \`{database_name}\`.\`{table_name}\`", message=f"{message}" ) else: retry(node.query, timeout=timeout, delay=3)( - f"EXISTS {database_name}.\`{table_name}\`", message=f"{message}" + f"EXISTS \`{database_name}\`.\`{table_name}\`", message=f"{message}" ) diff --git a/sink-connector/tests/integration/tests/steps/mysql/mysql.py b/sink-connector/tests/integration/tests/steps/mysql/mysql.py index d353726b8..70be1136f 100644 --- a/sink-connector/tests/integration/tests/steps/mysql/mysql.py +++ b/sink-connector/tests/integration/tests/steps/mysql/mysql.py @@ -18,11 +18,11 @@ def generate_special_case_names(num_names, max_length=64): "Transaction", ] - special_chars = "_$" + special_chars = "_-()" utf8_chars = "áéíóúñüç" chinese_characters = "中文女" - punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_{|}~""" + punctuation = r"""!#$%&()*+,-.<=>?@[]^_{|}~""" def generate_table_name(length): """Generate a random table name of a given length.""" @@ -30,9 +30,9 @@ def generate_table_name(length): string.ascii_letters + string.digits + special_chars - + utf8_chars + # + utf8_chars + chinese_characters - + punctuation + # + punctuation ) return "".join(random.choice(allowed_chars) for _ in range(length)) @@ -41,6 +41,14 @@ def generate_table_name(length): # Add reserved keywords table_names.update(reserved_keywords) + base_name = generate_table_name(random.randint(1, max_length)).lower() + case_variation_name = "".join( + random.choice([char.upper(), char.lower()]) for char in base_name + ) + + table_names.add(base_name) + table_names.add(case_variation_name) + while len(table_names) < num_names: length = random.randint(1, max_length) name = generate_table_name(length) @@ -131,12 +139,12 @@ def create_mysql_database(self, node=None, database_name=None): try: with Given(f"I create MySQL database {database_name}"): - node.query(rf"DROP DATABASE IF EXISTS {database_name};") - node.query(rf"CREATE DATABASE IF NOT EXISTS {database_name};") + node.query(rf"DROP DATABASE IF EXISTS \`{database_name}\`;") + node.query(rf"CREATE DATABASE IF NOT EXISTS \`{database_name}\`;") yield finally: with Finally(f"I delete MySQL database {database_name}"): - node.query(rf"DROP DATABASE IF EXISTS {database_name};") + node.query(rf"DROP DATABASE IF EXISTS \`{database_name}\`;") @TestStep(Given) @@ -168,7 +176,7 @@ def create_mysql_table( key = f"{primary_key} INT NOT NULL," with Given(f"I create MySQL table", description=name): - query = rf"CREATE TABLE IF NOT EXISTS {database_name}.\`{table_name}\` ({key}{columns}" + query = rf"CREATE TABLE IF NOT EXISTS \`{database_name}\`.\`{table_name}\` ({key}{columns}" if primary_key is not None: query += f", PRIMARY KEY ({primary_key}))" @@ -191,12 +199,37 @@ def create_mysql_table( "I clean up by deleting MySQL to ClickHouse replicated table", description={name}, ): - mysql_node.query(rf"DROP TABLE IF EXISTS {database_name}.\`{table_name}\`;") + mysql_node.query( + rf"DROP TABLE IF EXISTS \`{database_name}\`.\`{table_name}\`;" + ) clickhouse_node.query( - rf"DROP TABLE IF EXISTS {database_name}.\`{table_name}\` ON CLUSTER replicated_cluster;" + rf"DROP TABLE IF EXISTS \`{database_name}\`.\`{table_name}\` ON CLUSTER replicated_cluster;" ) +@TestStep(Given) +def create_sample_table(self, table_name, database=None, node=None): + """Create a sample table in MySQL.""" + if node is None: + node = self.context.cluster.node("mysql-master") + + if database is None: + database = "test" + + with By(f"creating a sample table {table_name} in MySQL"): + create_mysql_table( + table_name=table_name, + database_name=database, + mysql_node=node, + columns=f"name VARCHAR(255)", + ) + + with And("Inserting sample data"): + insert( + table_name=table_name, values="1,'test'", node=node, database_name=database + ) + + @TestStep def create_mysql_to_clickhouse_replicated_table( self, @@ -366,7 +399,9 @@ def insert(self, table_name, values, node=None, database_name=None): node = self.context.cluster.node("mysql-master") with When("I insert data into MySQL table"): - node.query(rf"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});") + node.query( + rf"INSERT INTO \`{database_name}\`.\`{table_name}\` VALUES ({values});" + ) @TestStep(Given) From 0d8e7555fc4268ab48f58fbd169ba22d2df58d53 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:32:51 +0400 Subject: [PATCH 30/44] add database name to inserts --- sink-connector/tests/integration/tests/replication.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sink-connector/tests/integration/tests/replication.py b/sink-connector/tests/integration/tests/replication.py index bfa31b803..4c9c8471f 100644 --- a/sink-connector/tests/integration/tests/replication.py +++ b/sink-connector/tests/integration/tests/replication.py @@ -76,12 +76,15 @@ def auto_create_table( with When("I insert values into the table"): if not multiple_inserts: table_values = f"{generate_sample_mysql_value('INT')},{generate_sample_mysql_value(column_datatype)}" - insert(table_name=table_name, values=table_values) + insert( + table_name=table_name, values=table_values, database_name=database + ) else: for _ in range(10): insert( table_name=table_name, values=f"{generate_sample_mysql_value('INT')}, {generate_sample_mysql_value(column_datatype)}", + database_name=database, ) if not validate_values: table_values = "" @@ -230,7 +233,7 @@ def drop_column_on_source(self, database): @TestScenario -def add_primary_key_on_a_database(self, database): +def add_primary_key_on_a_database(self): """Check that the primary key is added to the table when we add a primary key on a database.""" table_name = f"table_{getuid()}" column = "col1" @@ -341,7 +344,6 @@ def deletes(self): Scenario(test=delete_specific_records)(database=database) - @TestSuite def updates(self): """Check that updates are replicated to the destination.""" From b52593251402c9d1a3ce7c238d3a9aa4e2c84f14 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:33:24 +0400 Subject: [PATCH 31/44] update multiple databases suite --- .../integration/tests/multiple_databases.py | 59 +++++++++++++++---- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/sink-connector/tests/integration/tests/multiple_databases.py b/sink-connector/tests/integration/tests/multiple_databases.py index 51a25a324..ee06b8132 100644 --- a/sink-connector/tests/integration/tests/multiple_databases.py +++ b/sink-connector/tests/integration/tests/multiple_databases.py @@ -1,10 +1,17 @@ from testflows.core import * + +from integration.helpers.common import getuid from integration.tests.replication import replication +from integration.tests.steps.configurations import init_sink_connector_auto_created from integration.tests.steps.mysql.mysql import ( create_mysql_database, generate_special_case_names, + create_sample_table, +) +from integration.tests.steps.clickhouse import ( + create_clickhouse_database, + check_if_table_was_created, ) -from integration.tests.steps.clickhouse import create_clickhouse_database @TestStep(Given) @@ -32,6 +39,32 @@ def create_source_and_destination_databases(self, database_name=None): create_mysql_database(database_name=database_name) +@TestScenario +def check_database_creation(self): + """Check if the databases are created.""" + databases = generate_special_case_names( + self.context.number_of_databases, max_length=63 + ) + table_name = f"table_{getuid()}" + + with Given("I create databases with different combinations in names"): + for database in databases: + create_source_and_destination_databases(database_name=database) + + with And("I create a sample table on each of these databases"): + for database in databases: + with By("initializing sink connector with the database"): + init_sink_connector_auto_created( + topics=f"SERVER5432.{database}.{table_name}" + ) + with And("creating a sample table on that database"): + create_sample_table(database=database, table_name=table_name) + + with Then("I validate that the table was created on each of these databases"): + for database in databases: + check_if_table_was_created(database_name=database, table_name=table_name) + + @TestScenario def create_databases_with_special_names(self): """Create databases with special names on source and destination manually.""" @@ -39,26 +72,31 @@ def create_databases_with_special_names(self): "auto.create.tables": "true", "auto.create.tables.replicated": "false", } - databases = generate_special_case_names(self.context.number_of_databases) - - for database in databases: - create_source_and_destination_databases(database_name=rf"\`{database}\`") + databases = generate_special_case_names( + self.context.number_of_databases, max_length=63 + ) + with Given("I create databases with special characters in the name"): + for database in databases: + create_source_and_destination_databases(database_name=rf"\`{database}\`") - replication(databases=databases, update=update_config) + Check(test=replication)(databases=databases, update=update_config) @TestScenario def auto_create_databases_with_special_names(self): """Auto create databases with special characters in the name on source and wait and check it's created on destination.""" - databases = generate_special_case_names(self.context.number_of_databases) + databases = generate_special_case_names( + self.context.number_of_databases, max_length=63 + ) update_config = { "auto.create.tables": "true", "auto.create.tables.replicated": "false", } - for database in databases: - create_source_database(database_name=rf"\`{database}\`") + with Given("I create databases with special characters in the name"): + for database in databases: + create_source_database(database_name=rf"\`{database}\`") - replication(databases=databases, update=update_config) + Check(test=replication)(databases=databases, update=update_config) @TestFeature @@ -67,5 +105,6 @@ def feature(self, number_of_databases=100): """Validate sink connector with multiple databases.""" self.context.number_of_databases = number_of_databases + Scenario(run=check_database_creation) Scenario(run=create_databases_with_special_names) Scenario(run=auto_create_databases_with_special_names) From dac9c16c40798bcf197906516d1c75998c329b7d Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:33:38 +0400 Subject: [PATCH 32/44] update testflows version in requirements.txt --- sink-connector/tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/requirements.txt b/sink-connector/tests/integration/requirements.txt index fdac85095..7177ca226 100644 --- a/sink-connector/tests/integration/requirements.txt +++ b/sink-connector/tests/integration/requirements.txt @@ -1,4 +1,4 @@ -testflows==2.1.5 +testflows==2.4.10 python-dateutil==2.9.0 numpy==1.26.4 pyarrow==16.1.0 From e2da8467c41f20ce852cc7df7fadeb5dc1de7bc8 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:34:00 +0400 Subject: [PATCH 33/44] check that service is healthy for mysql --- sink-connector/tests/integration/env/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sink-connector/tests/integration/env/docker-compose.yml b/sink-connector/tests/integration/env/docker-compose.yml index 469065bfc..6cf4167d0 100644 --- a/sink-connector/tests/integration/env/docker-compose.yml +++ b/sink-connector/tests/integration/env/docker-compose.yml @@ -134,4 +134,6 @@ services: condition: service_healthy zookeeper: condition: service_healthy + mysql-master: + condition: service_healthy From ba8f2cf4366701fa5068a72cc9b1c15f6e13cf2f Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:34:13 +0400 Subject: [PATCH 34/44] fix import path --- sink-connector/tests/integration/helpers/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/helpers/common.py b/sink-connector/tests/integration/helpers/common.py index dd480c55c..ff214b0de 100644 --- a/sink-connector/tests/integration/helpers/common.py +++ b/sink-connector/tests/integration/helpers/common.py @@ -590,7 +590,7 @@ def set_envs_on_node(self, envs, node=None): node.command(f"unset {key}", exitcode=0) -from helpers.cluster import Cluster +from integration.helpers.cluster import Cluster @TestStep(Given) From 1f3d16db9659a854bcd5740df622f51a7bf9e55b Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:34:27 +0400 Subject: [PATCH 35/44] add query to the error message --- sink-connector/tests/integration/helpers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/helpers/cluster.py b/sink-connector/tests/integration/helpers/cluster.py index fe2560ce8..f99a6b7b6 100755 --- a/sink-connector/tests/integration/helpers/cluster.py +++ b/sink-connector/tests/integration/helpers/cluster.py @@ -884,7 +884,7 @@ def query( if "Exception:" in r.output: if raise_on_exception: raise QueryRuntimeException(r.output) - assert False, error(r.output) + assert False, f"{error(r.output)}\nfor query {sql}" elif "ERROR" in r.output: if raise_on_exception: raise QueryRuntimeException(r.output) From 1dfa8b1183d4c6b657e9b8ca6f807ca75b25b4be Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:34:56 +0400 Subject: [PATCH 36/44] add output style to CI/CD runs --- .../testflows-sink-connector-kafka.yml | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index 1e6650f13..27a0b36e6 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -12,6 +12,10 @@ on: description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'" type: string default: docker://clickhouse/clickhouse-server:23.3 + output_format: + description: "Testflows output style." + type: string + default: nice-new-fails secrets: DOCKERHUB_USERNAME: required: false @@ -38,6 +42,26 @@ on: custom_run_name: description: 'Custom run name (optional)' required: false + output_format: + description: "Testflows output style." + type: choice + options: + - nice-new-fails + - brisk-new-fails + - plain-new-fails + - pnice-new-fails + - new-fails + - classic + - nice + - fails + - slick + - brisk + - quiet + - short + - manual + - dots + - progress + - raw env: SINK_CONNECTOR_IMAGE: ${{ inputs.SINK_CONNECTOR_IMAGE }} @@ -90,7 +114,7 @@ jobs: - name: Run testflows tests working-directory: sink-connector/tests/integration - run: python3 -u regression.py --only "/regression/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + run: python3 -u regression.py --only "/regression/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --output ${{ inputs.output_format }} --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log - name: Create tfs results report if: always() From 0cffe6dd3aaae4ccdef3ee7c17ba1bad69948600 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:36:13 +0400 Subject: [PATCH 37/44] add output style to CI/CD runs --- .../testflows-sink-connector-lightweight.yml | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index db7e320df..642640129 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -12,6 +12,10 @@ on: description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'" type: string default: docker://clickhouse/clickhouse-server:23.3 + output_format: + description: "Testflows output style." + type: string + default: nice-new-fails secrets: DOCKERHUB_USERNAME: required: false @@ -38,6 +42,26 @@ on: custom_run_name: description: 'Custom run name (optional)' required: false + output_format: + description: "Testflows output style." + type: choice + options: + - nice-new-fails + - brisk-new-fails + - plain-new-fails + - pnice-new-fails + - new-fails + - classic + - nice + - fails + - slick + - brisk + - quiet + - short + - manual + - dots + - progress + - raw env: SINK_CONNECTOR_IMAGE: ${{ inputs.SINK_CONNECTOR_IMAGE }} @@ -91,7 +115,7 @@ jobs: - name: Run testflows tests working-directory: sink-connector-lightweight/tests/integration - run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --output ${{ inputs.output_format }} --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log - name: Create tfs results report if: always() From c42eedff0e0434b32690cdde9d371f3c0bf72f3f Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 15:44:33 +0400 Subject: [PATCH 38/44] update testflows version for lightweight --- sink-connector-lightweight/tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector-lightweight/tests/integration/requirements.txt b/sink-connector-lightweight/tests/integration/requirements.txt index fdac85095..7177ca226 100644 --- a/sink-connector-lightweight/tests/integration/requirements.txt +++ b/sink-connector-lightweight/tests/integration/requirements.txt @@ -1,4 +1,4 @@ -testflows==2.1.5 +testflows==2.4.10 python-dateutil==2.9.0 numpy==1.26.4 pyarrow==16.1.0 From 098f075d399543b3be3d55896b3552c20cec5cb7 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 13 Aug 2024 16:39:02 +0400 Subject: [PATCH 39/44] fix output format --- .github/workflows/testflows-sink-connector-kafka.yml | 2 +- .github/workflows/testflows-sink-connector-lightweight.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index 27a0b36e6..c247dadcf 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -114,7 +114,7 @@ jobs: - name: Run testflows tests working-directory: sink-connector/tests/integration - run: python3 -u regression.py --only "/regression/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --output ${{ inputs.output_format }} --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + run: python3 -u regression.py --only "/regression/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log - name: Create tfs results report if: always() diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index 642640129..4d661ec97 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -115,7 +115,7 @@ jobs: - name: Run testflows tests working-directory: sink-connector-lightweight/tests/integration - run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --output ${{ inputs.output_format }} --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log - name: Create tfs results report if: always() From 2d3aabeefa41d609ea4f3790debdffd07d70fd47 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 27 Aug 2024 14:16:47 +0400 Subject: [PATCH 40/44] check if types suite works --- sink-connector/tests/integration/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/regression.py b/sink-connector/tests/integration/regression.py index 41e73720b..35b4dea08 100755 --- a/sink-connector/tests/integration/regression.py +++ b/sink-connector/tests/integration/regression.py @@ -121,8 +121,8 @@ def regression( Feature(run=load("tests.truncate", "feature")) Feature(run=load("tests.deduplication", "feature")) Feature(run=load("tests.primary_keys", "feature")) - Feature(run=load("tests.virtual_columns", "feature")) Feature(run=load("tests.columns_inconsistency", "feature")) + Feature(run=load("tests.types", "feature")) Feature(run=load("tests.multiple_databases", "feature")) From 242c3f55e33492abf7f0dbcbe1e6737c521edf11 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 27 Aug 2024 14:53:54 +0400 Subject: [PATCH 41/44] fix test levels --- .../tests/integration/tests/types.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sink-connector/tests/integration/tests/types.py b/sink-connector/tests/integration/tests/types.py index a30ae9bf3..d326667a4 100644 --- a/sink-connector/tests/integration/tests/types.py +++ b/sink-connector/tests/integration/tests/types.py @@ -48,7 +48,7 @@ def check_datatype_replication( ) -@TestOutline(Feature) +@TestOutline(Scenario) @Examples( "mysql_type ch_type values ch_values nullable", [ @@ -83,7 +83,7 @@ def decimal(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestOutline(Feature) +@TestOutline(Scenario) @Examples( "mysql_type ch_type values ch_values nullable", [ @@ -112,7 +112,7 @@ def date_time(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestOutline(Feature) +@TestOutline(Scenario) # @Repeat(3) @Examples( "mysql_type ch_type values ch_values nullable", @@ -179,7 +179,7 @@ def integer_types(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestOutline(Feature) +@TestOutline(Scenario) @Examples( "mysql_type ch_type values ch_values nullable", [ @@ -208,7 +208,7 @@ def string(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestOutline(Feature) +@TestOutline(Scenario) @Examples( "mysql_type ch_type values ch_values nullable", [ @@ -244,7 +244,7 @@ def blob(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestOutline(Feature) +@TestOutline(Scenario) @Examples( "mysql_type ch_type values ch_values nullable", [ @@ -272,7 +272,7 @@ def binary(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestOutline(Feature) +@TestOutline(Scenario) @Examples( "mysql_type ch_type values ch_values nullable", [ @@ -297,12 +297,12 @@ def enum(self, mysql_type, ch_type, values, ch_values, nullable): ) -@TestModule +@TestFeature @Requirements( RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_DataTypes_Nullable("1.0") ) @Name("types") -def module(self): +def feature(self): """Verify correct replication of all supported MySQL data types.""" with Given("I enable debezium and sink connectors after kafka starts up"): @@ -310,7 +310,7 @@ def module(self): with Pool(1) as executor: try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() + for scenario in loads(current_module(), Scenario): + Feature(test=scenario, parallel=True, executor=executor)() finally: join() From 1d8c34e923ad593d340143266bd48f316762df0d Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 27 Aug 2024 16:13:01 +0400 Subject: [PATCH 42/44] move testflows lightweight rrmt test to nice-new-fails output --- .github/workflows/testflows-sink-connector-lightweight.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index 4d661ec97..6b62e7364 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -193,7 +193,7 @@ jobs: - name: Run testflows tests working-directory: sink-connector-lightweight/tests/integration - run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log - name: Create tfs results report if: always() From 0b849921897c02fa578045521674bd7a9cbbcfbb Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 2 Sep 2024 15:56:28 +0400 Subject: [PATCH 43/44] fix syntax warning --- sink-connector/tests/integration/tests/steps/clickhouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/tests/integration/tests/steps/clickhouse.py b/sink-connector/tests/integration/tests/steps/clickhouse.py index 08f40c02e..29cbef374 100644 --- a/sink-connector/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector/tests/integration/tests/steps/clickhouse.py @@ -192,7 +192,7 @@ def check_if_table_was_created( ) else: retry(node.query, timeout=timeout, delay=3)( - f"EXISTS \`{database_name}\`.\`{table_name}\`", message=f"{message}" + rf"EXISTS \`{database_name}\`.\`{table_name}\`", message=f"{message}" ) From bda3400213c15d709f5f3bbf52276ff188df74e3 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Mon, 2 Sep 2024 16:05:49 +0400 Subject: [PATCH 44/44] fix syntax warning --- .../tests/integration/tests/is_deleted.py | 6 ++--- .../integration/tests/multiple_databases.py | 24 +++++++++---------- .../tests/integration/tests/steps/mysql.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/sink-connector-lightweight/tests/integration/tests/is_deleted.py b/sink-connector-lightweight/tests/integration/tests/is_deleted.py index b35e722aa..4ba56fd0c 100644 --- a/sink-connector-lightweight/tests/integration/tests/is_deleted.py +++ b/sink-connector-lightweight/tests/integration/tests/is_deleted.py @@ -16,9 +16,9 @@ def create_table_with_is_deleted( clickhouse_node = self.context.clickhouse_node if not backticks: - columns = "col1 varchar(255), col2 int, " + columns = r"col1 varchar(255), col2 int, " else: - columns = "\`col1\` varchar(255), \`col2\` int, " + columns = r"\`col1\` varchar(255), \`col2\` int, " with By( f"creating a {table_name} table with is_deleted column and {datatype} datatype" @@ -196,7 +196,7 @@ def column_with_is_deleted_backticks(self): table_name = "tb_" + getuid() with Given(f"I create the {table_name} table and populate it with data"): - create_table_with_is_deleted(table_name=table_name, column="\`is_deleted\`") + create_table_with_is_deleted(table_name=table_name, column=r"\`is_deleted\`") with Then("I check that the data was inserted correctly into the ClickHouse table"): for retry in retries(timeout=40, delay=1): diff --git a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py index 9fe2dc1ae..7197ce7c9 100644 --- a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py +++ b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py @@ -641,10 +641,10 @@ def different_database_names(self): def different_database_names_with_source_backticks(self): """Check that the tables are replicated when we have source and destination databases with different names and source database name contains backticks.""" database_map = { - "\`mysql1\`": "ch1", - "\`mysql2\`": "ch2", - "\`mysql3\`": "ch3", - "\`mysql4\`": "ch4", + r"\`mysql1\`": r"ch1", + r"\`mysql2\`": r"ch2", + r"\`mysql3\`": r"ch3", + r"\`mysql4\`": r"ch4", } check_different_database_names(database_map=database_map) @@ -653,10 +653,10 @@ def different_database_names_with_source_backticks(self): def different_database_names_with_destination_backticks(self): """Check that the tables are replicated when we have source and destination databases with different names and destination database name contains backticks.""" database_map = { - "mysql1": "\`ch1\`", - "mysql2": "\`ch2\`", - "mysql3": "\`ch3\`", - "mysql4": "\`ch4\`", + r"mysql1": r"\`ch1\`", + r"mysql2": r"\`ch2\`", + r"mysql3": r"\`ch3\`", + r"mysql4": r"\`ch4\`", } check_different_database_names(database_map=database_map) @@ -665,10 +665,10 @@ def different_database_names_with_destination_backticks(self): def different_database_names_with_backticks(self): """Check that the tables are replicated when we have source and destination databases with the same names and they contain backticks.""" database_map = { - "\`mysql1\`": "\`ch1\`", - "\`mysql2\`": "\`ch2\`", - "\`mysql3\`": "\`ch3\`", - "\`mysql4\`": "\`ch4\`", + r"\`mysql1\`": r"\`ch1\`", + r"\`mysql2\`": r"\`ch2\`", + r"\`mysql3\`": r"\`ch3\`", + r"\`mysql4\`": r"\`ch4\`", } check_different_database_names(database_map=database_map) diff --git a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py index e451522e1..689deebdd 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py @@ -317,7 +317,7 @@ def insert(self, table_name, values, node=None, database_name=None): node = self.context.cluster.node("mysql-master") with When("I insert data into MySQL table"): - node.query(f"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});") + node.query(rf"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});") @TestStep(Given)