diff --git a/sink-connector-lightweight/dependency-reduced-pom.xml b/sink-connector-lightweight/dependency-reduced-pom.xml index a5d22b3c7..6926b2a96 100644 --- a/sink-connector-lightweight/dependency-reduced-pom.xml +++ b/sink-connector-lightweight/dependency-reduced-pom.xml @@ -162,10 +162,6 @@ oracle-xe org.testcontainers - - junit-platform-launcher - org.junit.platform - junit-jupiter org.junit.jupiter @@ -207,7 +203,7 @@ org.testcontainers testcontainers - 1.19.1 + 1.20.4 test @@ -231,7 +227,7 @@ org.testcontainers jdbc - 1.19.1 + 1.20.4 test @@ -243,37 +239,37 @@ org.testcontainers junit-jupiter - 1.19.1 + 1.20.4 test org.testcontainers clickhouse - 1.19.1 + 1.20.4 test org.testcontainers mysql - 1.19.1 + 1.20.4 test org.testcontainers postgresql - 1.19.1 + 1.20.4 test org.testcontainers mariadb - 1.19.1 + 1.20.4 test org.testcontainers mongodb - 1.19.1 + 1.20.4 test @@ -289,10 +285,6 @@ 5.8.1 test - - junit-platform-engine - org.junit.platform - junit-jupiter-api org.junit.jupiter @@ -317,10 +309,38 @@ junit-jupiter-api org.junit.jupiter + + + + org.junit.platform + junit-platform-launcher + 1.8.2 + test + + + apiguardian-api + org.apiguardian + + + + + org.junit.platform + junit-platform-engine + 1.8.2 + test + - junit-platform-launcher + opentest4j + org.opentest4j + + + junit-platform-commons org.junit.platform + + apiguardian-api + org.apiguardian + @@ -356,7 +376,7 @@ 2.14.0.Final quarkus-bom - 1.19.1 + 1.20.4 3.0.0-M7 3.8.0 UTF-8 diff --git a/sink-connector-lightweight/docker/config.yml b/sink-connector-lightweight/docker/config.yml index a9738f1aa..6be2ee681 100644 --- a/sink-connector-lightweight/docker/config.yml +++ b/sink-connector-lightweight/docker/config.yml @@ -29,7 +29,7 @@ database.server.name: "ER54" # database.include.list An optional list of regular expressions that match database names to be monitored; # any database name not included in the whitelist will be excluded from monitoring. By default all databases will be monitored. -database.include.list: test +database.include.list: sbtest # table.include.list An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored; table.include.list: "" @@ -179,6 +179,8 @@ ignore.ddl.regex: "(?i)(ANALYZE PARTITION).*" #Metrics (Prometheus target), required for Grafana Dashboard metrics.enable: "true" +connection.pool.max.size: 1000 + # Skip schema history capturing, use the following configuration # to reduce slow startup when replicating dbs with large number of tables #schema.history.internal.store.only.captured.tables.ddl: "true" @@ -189,4 +191,4 @@ use.nongraceful.disconnect: "true" database.keep.alive.interval.`ms: "30000" #Send keepalive every 30 seconds database.connection.reconnect.backoff.ms: "1000" database.connection.reconnect.backoff.max.ms: "10000" -database.ssl.mode: "disabled" \ No newline at end of file +database.ssl.mode: "disabled" diff --git a/sink-connector-lightweight/pom.xml b/sink-connector-lightweight/pom.xml index 411c24da7..9705ae742 100644 --- a/sink-connector-lightweight/pom.xml +++ b/sink-connector-lightweight/pom.xml @@ -15,7 +15,7 @@ UTF-8 3.0.0.Final 5.9.1 - 1.19.1 + 1.20.4 3.1.1 UTF-8 UTF-8 @@ -233,6 +233,11 @@ + + com.zaxxer + HikariCP + 6.0.0 + org.antlr antlr4-runtime @@ -334,6 +339,18 @@ test + + org.junit.platform + junit-platform-launcher + 1.8.2 + test + + + org.junit.platform + junit-platform-engine + 1.8.2 + test + org.apache.tomcat annotations-api diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java index 3f102b4ed..8d428401e 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java @@ -2,7 +2,6 @@ import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; @@ -10,6 +9,7 @@ import com.altinity.clickhouse.sink.connector.common.Metrics; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; import com.altinity.clickhouse.sink.connector.db.DBMetadata; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.db.operations.ClickHouseAlterTable; import com.altinity.clickhouse.sink.connector.executor.ClickHouseBatchExecutor; import com.altinity.clickhouse.sink.connector.executor.ClickHouseBatchRunnable; @@ -17,7 +17,6 @@ import com.altinity.clickhouse.sink.connector.executor.DebeziumOffsetManagement; import com.altinity.clickhouse.sink.connector.model.ClickHouseStruct; import com.altinity.clickhouse.sink.connector.model.DBCredentials; -import com.clickhouse.jdbc.ClickHouseConnection; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.debezium.config.CommonConnectorConfig; @@ -41,6 +40,7 @@ import org.json.simple.parser.ParseException; import java.io.IOException; +import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -89,7 +89,7 @@ public class DebeziumChangeEventCapture { DebeziumEngine> engine; // Keep one clickhouse connection. - private ClickHouseConnection conn; + private Connection conn; ClickHouseBatchWriter singleThreadedWriter; @@ -197,7 +197,8 @@ private String getDatabaseName(SourceRecord sr) { private BaseDbWriter createWriter(ClickHouseSinkConnectorConfig config, String databaseName) { DBCredentials dbCredentials = parseDBConfiguration(config); String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), databaseName); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", dbCredentials.getUserName(), dbCredentials.getPassword(), config); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, dbCredentials.getUserName(), dbCredentials.getPassword(), + databaseName, config); return new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); } @@ -270,11 +271,17 @@ private ClickHouseStruct processEveryChangeRecord(Properties props, ChangeEvent< if (DDL != null && DDL.isEmpty() == false) { log.info("***** DDL received, Flush all existing records"); - this.executor.shutdown(); - this.executor.awaitTermination(60, TimeUnit.SECONDS); +// this.executor.shutdown(); +// this.executor.awaitTermination(60, TimeUnit.SECONDS); +// //HikariDbSource.closeDatabaseConnection(BaseDbWriter.SYSTEM_DB); + this.executor.pause(); + //this.writer = null; + //this.writer = null; performDDLOperation(DDL, props, sr, config, recordCommitter, record, lastRecordInBatch); - setupProcessingThread(config); + this.executor.resume(); + //HikariDbSource.closeDatabaseConnection(BaseDbWriter.SYSTEM_DB); + //setupProcessingThread(config); } } else { @@ -383,10 +390,11 @@ private void createDatabaseForDebeziumStorage(ClickHouseSinkConnectorConfig conf DBCredentials dbCredentials = parseDBConfiguration(config); String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), - "system"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", dbCredentials.getUserName(), dbCredentials.getPassword(), config); + BaseDbWriter.SYSTEM_DB); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + dbCredentials.getUserName(), dbCredentials.getPassword(), BaseDbWriter.SYSTEM_DB, config); BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - "system", dbCredentials.getUserName(), + BaseDbWriter.SYSTEM_DB, dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); Pair tableNameDatabaseName = getDebeziumOffsetStorageDatabaseName(props); @@ -394,7 +402,7 @@ private void createDatabaseForDebeziumStorage(ClickHouseSinkConnectorConfig conf String createDbQuery = String.format("create database if not exists %s", databaseName); log.info("CREATING DEBEZIUM STORAGE Database: " + createDbQuery); - writer.executeQuery(createDbQuery); + writer.executeSystemQuery(createDbQuery); break; } catch (Exception e) { @@ -422,13 +430,14 @@ private void createSchemaHistoryTable(ClickHouseSinkConnectorConfig config, Prop DBCredentials dbCredentials = parseDBConfiguration(config); String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), "system"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1",dbCredentials.getUserName(), dbCredentials.getPassword(), config); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + dbCredentials.getUserName(), dbCredentials.getPassword(), BaseDbWriter.SYSTEM_DB, config); BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), "system", dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); try { - writer.executeQuery(createSchemaHistoryTable); + writer.executeSystemQuery(createSchemaHistoryTable); } catch(Exception e) { log.error("Error creating schema history table", e); } @@ -449,7 +458,8 @@ private void createViewForShowReplicaStatus(ClickHouseSinkConnectorConfig config String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), "system"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1",dbCredentials.getUserName(), dbCredentials.getPassword(), config); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + dbCredentials.getUserName(), dbCredentials.getPassword(), BaseDbWriter.SYSTEM_DB, config); BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), "system", dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); @@ -462,7 +472,7 @@ private void createViewForShowReplicaStatus(ClickHouseSinkConnectorConfig config // Remove quotes. formattedView = formattedView.replace("\"", ""); try { - writer.executeQuery(formattedView); + writer.executeSystemQuery(formattedView); } catch(Exception e) { log.error("**** Error creating VIEW **** " + formattedView); } @@ -539,8 +549,9 @@ public String getDebeziumStorageStatus(ClickHouseSinkConnectorConfig config, Pro log.error("**** Connection to ClickHouse is not established, re-initiating ****"); String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), databaseName); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - dbCredentials.getUserName(), dbCredentials.getPassword(), config); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + dbCredentials.getUserName(), dbCredentials.getPassword(), + BaseDbWriter.SYSTEM_DB, config); writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); @@ -956,6 +967,7 @@ DBCredentials parseDBConfiguration(ClickHouseSinkConnectorConfig config) { */ private void setupProcessingThread(ClickHouseSinkConnectorConfig config) { + if(config.getBoolean(ClickHouseSinkConnectorConfigVariables.SINGLE_THREADED.toString())) { log.info("********* Running in Single Threaded mode *********"); singleThreadedWriter = new ClickHouseBatchWriter(config, new HashMap()); diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java index 903f92878..8c2a85544 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java @@ -51,7 +51,7 @@ public void deleteOffsetStorageRow(String offsetKey, // String connectorName = config.getString("connector.name"); String debeziumStorageStatusQuery = String.format("delete from %s where offset_key='%s'" , tableName, offsetKey); - writer.executeQuery(debeziumStorageStatusQuery); + writer.executeSystemQuery(debeziumStorageStatusQuery); } /** @@ -67,7 +67,7 @@ public void deleteSchemaHistoryTable(String offsetKey, String debeziumStorageStatusQuery = String.format("delete from `%s` where JSONExtractRaw(JSONExtractRaw(history_data,'source'), 'server')='%s'" , tableName, offsetKey); log.info("Deleting schema history table query: " + debeziumStorageStatusQuery); - writer.executeQuery(debeziumStorageStatusQuery); + writer.executeSystemQuery(debeziumStorageStatusQuery); } /** * Function to get the latest timestamp of the record in the table @@ -81,7 +81,7 @@ public String getDebeziumLatestRecordTimestamp(Properties props, BaseDbWriter wr JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); String debeziumLatestRecordTimestampQuery = String.format("select max(record_insert_ts) from %s" , tableName); - return writer.executeQuery(debeziumLatestRecordTimestampQuery); + return writer.executeSystemQuery(debeziumLatestRecordTimestampQuery); } public String getDebeziumStorageStatusQuery( @@ -92,7 +92,7 @@ public String getDebeziumStorageStatusQuery( String offsetKey = getOffsetKey(props); // String connectorName = config.getString("connector.name"); String debeziumStorageStatusQuery = String.format("select offset_val from %s where offset_key='%s'" , tableName, offsetKey); - return writer.executeQuery(debeziumStorageStatusQuery); + return writer.executeSystemQuery(debeziumStorageStatusQuery); } /** diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMongoIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMongoIT.java index 724f393ee..a048d1512 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMongoIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMongoIT.java @@ -1,10 +1,8 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.config.EnvironmentConfigurationService; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.mongodb.MongoException; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; @@ -15,6 +13,7 @@ import org.bson.types.ObjectId; import org.junit.ClassRule; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.DockerComposeContainer; @@ -35,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference; @Testcontainers +@Disabled public class ClickHouseDebeziumEmbeddedMongoIT { @Container @@ -112,6 +112,7 @@ public void testDataTypesDB() throws Exception { // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); //writer.getConnection().close(); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresDecoderBufsDockerIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresDecoderBufsDockerIT.java index a9f044582..302ba5267 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresDecoderBufsDockerIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresDecoderBufsDockerIT.java @@ -2,13 +2,10 @@ import static com.altinity.clickhouse.debezium.embedded.PostgresProperties.getDefaultProperties; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.junit.Assert; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.testcontainers.Testcontainers; @@ -19,7 +16,6 @@ import org.testcontainers.utility.DockerImageName; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -90,13 +86,8 @@ public void testDecoderBufsPlugin() throws Exception { Thread.sleep(10000);// Thread.sleep(50000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + Map tmColumns = writer.getColumnsDataTypesForTable("tm"); Assert.assertTrue(tmColumns.size() == 22); Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); @@ -106,7 +97,7 @@ public void testDecoderBufsPlugin() throws Exception { int tmCount = 0; - ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from tm").executeQuery(); + ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from public.tm").executeQuery(); while(chRs.next()) { tmCount = chRs.getInt(1); } @@ -119,5 +110,6 @@ public void testDecoderBufsPlugin() throws Exception { // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresPgoutputDockerIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresPgoutputDockerIT.java index 5df4eebec..3ee800ceb 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresPgoutputDockerIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedPostgresPgoutputDockerIT.java @@ -2,13 +2,10 @@ import static com.altinity.clickhouse.debezium.embedded.PostgresProperties.getDefaultProperties; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.junit.Assert; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.testcontainers.Testcontainers; @@ -18,6 +15,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.utility.DockerImageName; +import java.sql.Connection; import java.sql.ResultSet; import java.util.HashMap; import java.util.Map; @@ -94,13 +92,8 @@ public void testPgOutputPlugin() throws Exception { // Create connection. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + Map tmColumns = writer.getColumnsDataTypesForTable("tm"); Assert.assertTrue(tmColumns.size() == 22); Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); @@ -110,7 +103,7 @@ public void testPgOutputPlugin() throws Exception { int tmCount = 0; - ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from tm").executeQuery(); + ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from public.tm").executeQuery(); while(chRs.next()) { tmCount = chRs.getInt(1); } @@ -123,5 +116,7 @@ public void testPgOutputPlugin() throws Exception { // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java index fc7a82d38..d5f01b45a 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java @@ -2,6 +2,9 @@ import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; + import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.*; @@ -9,7 +12,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.Properties; - +import java.util.HashMap; public class ITCommon { static public Connection connectToMySQL(MySQLContainer mySqlContainer) { Connection conn = null; @@ -196,4 +199,31 @@ static public Properties getDebeziumPropertiesForSchemaOnly(MySQLContainer mySql props.setProperty("replica.status.view", "CREATE VIEW IF NOT EXISTS %s.show_replica_status AS SELECT now() - fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS seconds_behind_source, toDateTime(fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')), 'UTC') AS utc_time, fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS local_time FROM %s settings final=1"); return props; } + + + static public BaseDbWriter getDBWriter(ClickHouseContainer clickHouseContainer) { + + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees"); + Connection connection = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, clickHouseContainer.getUsername(), + clickHouseContainer.getPassword(), BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); + + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + + return writer; + } + + static public BaseDbWriter getDBWriter(ClickHouseContainer clickHouseContainer, String databaseName) { + + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + databaseName); + Connection connection = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, clickHouseContainer.getUsername(), + clickHouseContainer.getPassword(), BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); + + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + databaseName, clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + + return writer; + } } \ No newline at end of file diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MariaDBIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MariaDBIT.java index a1087b6a2..6c896dab5 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MariaDBIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MariaDBIT.java @@ -1,11 +1,9 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -21,7 +19,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -126,15 +123,7 @@ public void testMultipleDatabases() throws Exception { conn.close(); // Create connection to clickhouse and validate if the tables are replicated. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - // query clickhouse connection and get data for test_table1 and test_table2 - + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); ResultSet rs = writer.executeQueryWithResultSet("SELECT * FROM employees.audience"); // Validate the data @@ -156,6 +145,6 @@ public void testMultipleDatabases() throws Exception { writer.getConnection().close(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java index 6444b11a9..331d509e8 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java @@ -1,18 +1,12 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLBaseIT; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -23,7 +17,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -64,6 +57,12 @@ public void startContainers() throws InterruptedException { Thread.sleep(25000); } + @AfterEach + public void tearDown() { + mySqlContainer.stop(); + clickHouseContainer.stop(); + } + @Test public void testMySQLGeneratedColumns() throws Exception { AtomicReference engine = new AtomicReference<>(); @@ -97,11 +96,7 @@ public void testMySQLGeneratedColumns() throws Exception { conn.prepareStatement("insert into contacts(first_name, last_name, email) values('John', 'Doe', 'john.doe@gmail.com')").execute(); Thread.sleep(20000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); Map columnsToDataTypeMap = writer.getColumnsDataTypesForTable("contacts"); Assert.assertTrue(columnsToDataTypeMap.get("id").equalsIgnoreCase("Int32")); @@ -110,7 +105,7 @@ public void testMySQLGeneratedColumns() throws Exception { Assert.assertTrue(columnsToDataTypeMap.get("fullname").equalsIgnoreCase("Nullable(String)")); Assert.assertTrue(columnsToDataTypeMap.get("email").equalsIgnoreCase("String")); - ResultSet resultSet = writer.executeQueryWithResultSet("select fullname from contacts"); + ResultSet resultSet = writer.executeQueryWithResultSet("select fullname from employees.contacts"); boolean insertCheck = false; while (resultSet.next()) { insertCheck = true; @@ -129,5 +124,7 @@ public void testMySQLGeneratedColumns() throws Exception { } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java index 6712ae23f..edeff5d21 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java @@ -1,11 +1,9 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -19,8 +17,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -118,13 +114,7 @@ public void testMultipleDatabases() throws Exception { conn.close(); // Create connection to clickhouse and validate if the tables are replicated. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); // query clickhouse connection and get data for test_table1 and test_table2 @@ -148,6 +138,8 @@ public void testMultipleDatabases() throws Exception { writer.getConnection().close(); + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java index c8419da2d..4be466f7e 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java @@ -2,9 +2,9 @@ import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; @@ -20,7 +20,6 @@ import org.testcontainers.utility.DockerImageName; import java.sql.Connection; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -116,6 +115,8 @@ public void testAutoCreateTable(String clickHouseServerVersion) throws Exception // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerIT.java index 96ac528fc..47ccd2301 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerIT.java @@ -2,11 +2,9 @@ import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumOffsetStorage; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.junit.Assert; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -17,6 +15,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.utility.DockerImageName; +import java.sql.Connection; import java.sql.ResultSet; import java.util.HashMap; import java.util.Map; @@ -91,13 +90,8 @@ public void testDecoderBufsPlugin() throws Exception { Thread.sleep(10000);// Thread.sleep(50000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + Map tmColumns = writer.getColumnsDataTypesForTable("tm"); Assert.assertTrue(tmColumns.size() == 22); Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); @@ -107,7 +101,7 @@ public void testDecoderBufsPlugin() throws Exception { int tmCount = 0; - ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from tm").executeQuery(); + ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from public.tm").executeQuery(); while(chRs.next()) { tmCount = chRs.getInt(1); } @@ -134,5 +128,7 @@ public void testDecoderBufsPlugin() throws Exception { // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerWKeeperMapStorageIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerWKeeperMapStorageIT.java index 45ce26625..2ee206b5b 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerWKeeperMapStorageIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerWKeeperMapStorageIT.java @@ -2,24 +2,24 @@ import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumOffsetStorage; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.testcontainers.Testcontainers; import org.testcontainers.clickhouse.ClickHouseContainer; -import org.testcontainers.containers.*; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.utility.DockerImageName; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -52,8 +52,8 @@ public void startContainers() throws InterruptedException { .withPassword("password") .withClasspathResourceMapping("config_replicated.xml", "/etc/clickhouse-server/config.d/config.xml", BindMode.READ_ONLY) .withClasspathResourceMapping("macros.xml", "/etc/clickhouse-server/config.d/macros.xml", BindMode.READ_ONLY) - .withExposedPorts(8123) - .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); + .withExposedPorts(8123); + //.waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); clickHouseContainer.withNetwork(network).withNetworkAliases("clickhouse"); clickHouseContainer.start(); } @@ -114,13 +114,8 @@ public void testDecoderBufsPlugin() throws Exception { Thread.sleep(10000);// Thread.sleep(50000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + Map tmColumns = writer.getColumnsDataTypesForTable("tm"); Assert.assertTrue(tmColumns.size() == 22); Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); @@ -130,7 +125,7 @@ public void testDecoderBufsPlugin() throws Exception { int tmCount = 0; - ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from tm").executeQuery(); + ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from public.tm").executeQuery(); while(chRs.next()) { tmCount = chRs.getInt(1); } @@ -156,7 +151,7 @@ public void testDecoderBufsPlugin() throws Exception { Thread.sleep(15000); // Check if the clickhouse table is empty. - chRs = writer.getConnection().prepareStatement("select count(*) from tm").executeQuery(); + chRs = writer.getConnection().prepareStatement("select count(*) from public.tm").executeQuery(); while(chRs.next()) { tmCount = chRs.getInt(1); } @@ -168,6 +163,7 @@ public void testDecoderBufsPlugin() throws Exception { } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java index 2ea19ea19..0a20c4dd5 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java @@ -1,11 +1,9 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.junit.Assert; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -18,7 +16,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -103,14 +100,8 @@ public void testMultipleSchemaReplication() throws Exception { Thread.sleep(10000); - // Create connection. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + Map tmColumns = writer.getColumnsDataTypesForTable("tm"); Assert.assertTrue(tmColumns.size() == 22); Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); @@ -145,11 +136,11 @@ public void testMultipleSchemaReplication() throws Exception { Thread.sleep(10000); // ClickHouse, add ALIAS column to public.people - conn.createStatement().execute("ALTER TABLE public.people ADD COLUMN full_name String ALIAS concat('John', ' ', 'Doe');"); + writer.getConnection().createStatement().execute("ALTER TABLE public.people ADD COLUMN full_name String ALIAS concat('John', ' ', 'Doe');"); Thread.sleep(10000); // Add MATERIALIZED column to public.people - conn.createStatement().execute("ALTER TABLE public.people ADD COLUMN full_name_mat String MATERIALIZED toString(height_cm)"); + writer.getConnection().createStatement().execute("ALTER TABLE public.people ADD COLUMN full_name_mat String MATERIALIZED toString(height_cm)"); postgresConn2.createStatement().execute("insert into public.people (height_cm) values (200)"); Thread.sleep(20000); @@ -166,6 +157,6 @@ public void testMultipleSchemaReplication() throws Exception { } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java index 70fae4413..53b5dff33 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java @@ -1,12 +1,10 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -24,7 +22,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,8 +62,9 @@ public void startContainers() throws InterruptedException { .withPassword("password") .withClasspathResourceMapping("config_replicated.xml", "/etc/clickhouse-server/config.d/config.xml", BindMode.READ_ONLY) .withClasspathResourceMapping("macros.xml", "/etc/clickhouse-server/config.d/macros.xml", BindMode.READ_ONLY) - .withExposedPorts(8123) - .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); + .withExposedPorts(8123); + //.waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort()) + clickHouseContainer.withNetwork(network).withNetworkAliases("clickhouse"); clickHouseContainer.start(); } @@ -101,12 +99,9 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E Thread.sleep(30000); Connection conn = ITCommon.connectToMySQL(mySqlContainer); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - ResultSet rs = writer.executeQueryWithResultSet("show create table string_types_MEDIUMTEXT_utf8mb4"); + ResultSet rs = writer.executeQueryWithResultSet("show create table employees.string_types_MEDIUMTEXT_utf8mb4"); // Validate that all the tables are created. boolean resultValidated = false; while(rs.next()) { @@ -120,7 +115,7 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E boolean dataValidated = false; // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from string_types_MEDIUMTEXT_utf8mb4"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.string_types_MEDIUMTEXT_utf8mb4"); while(dateTimeResult.next()) { dataValidated = true; @@ -136,6 +131,8 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java index 2ec6a53b1..37b5388f3 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java @@ -1,13 +1,10 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -25,7 +22,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -66,8 +62,8 @@ public void startContainers() throws InterruptedException { .withPassword("password") .withClasspathResourceMapping("config_replicated.xml", "/etc/clickhouse-server/config.d/config.xml", BindMode.READ_ONLY) .withClasspathResourceMapping("macros.xml", "/etc/clickhouse-server/config.d/macros.xml", BindMode.READ_ONLY) - .withExposedPorts(8123) - .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); + .withExposedPorts(8123); + //.waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); clickHouseContainer.withNetwork(network).withNetworkAliases("clickhouse"); clickHouseContainer.start(); } @@ -103,12 +99,9 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E Thread.sleep(30000); Connection conn = ITCommon.connectToMySQL(mySqlContainer); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - ResultSet rs = writer.executeQueryWithResultSet("show create table string_types_MEDIUMTEXT_utf8mb4"); + ResultSet rs = writer.executeQueryWithResultSet("show create table employees.string_types_MEDIUMTEXT_utf8mb4"); // Validate that all the tables are created. boolean resultValidated = false; while(rs.next()) { @@ -122,7 +115,7 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E boolean dataValidated = false; // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from string_types_MEDIUMTEXT_utf8mb4"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.string_types_MEDIUMTEXT_utf8mb4"); while(dateTimeResult.next()) { dataValidated = true; @@ -150,7 +143,7 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E Thread.sleep(10000); // Verify on ClickHouse if the table is created. - rs = writer.executeQueryWithResultSet("show create table l1"); + rs = writer.executeQueryWithResultSet("show create table employees.l1"); // Validate that all the tables are created. resultValidated = false; while(rs.next()) { @@ -167,6 +160,8 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E conn.close(); // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java index f0fe12e1b..c1af97724 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java @@ -1,13 +1,10 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -25,7 +22,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -59,15 +55,16 @@ public void startContainers() throws InterruptedException { // clickHouseContainer.start(); Thread.sleep(15000); - clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:23.8") .asCompatibleSubstituteFor("clickhouse")) .withInitScript("init_clickhouse_it.sql") .withUsername("ch_user") .withPassword("password") .withClasspathResourceMapping("config_replicated.xml", "/etc/clickhouse-server/config.d/config.xml", BindMode.READ_ONLY) .withClasspathResourceMapping("macros.xml", "/etc/clickhouse-server/config.d/macros.xml", BindMode.READ_ONLY) - .withExposedPorts(8123) - .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); + .withExposedPorts(8123); + + // .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); clickHouseContainer.withNetwork(network).withNetworkAliases("clickhouse"); clickHouseContainer.start(); } @@ -106,12 +103,9 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E Thread.sleep(30000); Connection conn = ITCommon.connectToMySQL(mySqlContainer); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - ResultSet rs = writer.executeQueryWithResultSet("show create table string_types_MEDIUMTEXT_utf8mb4"); + ResultSet rs = writer.executeQueryWithResultSet("show create table employees.string_types_MEDIUMTEXT_utf8mb4"); // Validate that all the tables are created. boolean resultValidated = false; while(rs.next()) { @@ -125,7 +119,7 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E boolean dataValidated = false; // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from string_types_MEDIUMTEXT_utf8mb4"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.string_types_MEDIUMTEXT_utf8mb4"); while(dateTimeResult.next()) { dataValidated = true; @@ -142,6 +136,8 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java index 6cb2c1e9c..47b4da956 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java @@ -1,12 +1,10 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -24,8 +22,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -66,8 +62,8 @@ public void startContainers() throws InterruptedException { .withPassword("password") .withClasspathResourceMapping("config_replicated.xml", "/etc/clickhouse-server/config.d/config.xml", BindMode.READ_ONLY) .withClasspathResourceMapping("macros.xml", "/etc/clickhouse-server/config.d/macros.xml", BindMode.READ_ONLY) - .withExposedPorts(8123) - .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); + .withExposedPorts(8123); +// .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); clickHouseContainer.withNetwork(network).withNetworkAliases("clickhouse"); clickHouseContainer.start(); } @@ -104,12 +100,9 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E Thread.sleep(30000); Connection conn = ITCommon.connectToMySQL(mySqlContainer); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - ResultSet rs = writer.executeQueryWithResultSet("show create table string_types_MEDIUMTEXT_utf8mb4"); + ResultSet rs = writer.executeQueryWithResultSet("show create table employees.string_types_MEDIUMTEXT_utf8mb4"); // Validate that all the tables are created. boolean resultValidated = false; while(rs.next()) { @@ -123,7 +116,7 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E boolean dataValidated = false; // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from string_types_MEDIUMTEXT_utf8mb4"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.string_types_MEDIUMTEXT_utf8mb4"); while(dateTimeResult.next()) { dataValidated = true; @@ -140,6 +133,8 @@ public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws E } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/AbstractCDCBaseIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/AbstractCDCBaseIT.java new file mode 100644 index 000000000..c92d7bbb2 --- /dev/null +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/AbstractCDCBaseIT.java @@ -0,0 +1,45 @@ +package com.altinity.clickhouse.debezium.embedded.cdc; + +import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + + +@Testcontainers +public class AbstractCDCBaseIT { + + protected static MySQLContainer mySqlContainer; + protected static ClickHouseContainer clickHouseContainer; + static { + clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + .asCompatibleSubstituteFor("clickhouse")) + .withInitScript("init_clickhouse_it.sql") + .withUsername("ch_user") + .withPassword("password") + .withExposedPorts(8123); + + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") + .asCompatibleSubstituteFor("mysql")) + .withDatabaseName("employees").withUsername("root").withPassword("adminpass") + .withInitScript("alter_ddl_add_column.sql") + .withExtraHost("mysql-server", "0.0.0.0") + .waitingFor(new HttpWaitStrategy().forPort(3306)); + + BasicConfigurator.configure(); + mySqlContainer.start(); + clickHouseContainer.start(); + + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} + diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java index 9e729d3ce..1b2e5236a 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java @@ -4,19 +4,14 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -61,6 +56,11 @@ public void startContainers() throws InterruptedException { clickHouseContainer.start(); Thread.sleep(15000); } + @AfterEach() + public void stop() { + mySqlContainer.stop(); + clickHouseContainer.stop(); + } @Test public void testBatchRetryOnCHFailure() throws Exception { @@ -103,13 +103,14 @@ public void testBatchRetryOnCHFailure() throws Exception { // Check if Batch was inserted. String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection chConn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), + "employees",new ClickHouseSinkConnectorConfig(new HashMap<>())); BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME where Type = 'DATETIME-INSERT55'"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME where Type = 'DATETIME-INSERT55'"); boolean insertCheck = false; while(dateTimeResult.next()) { insertCheck = true; diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java index d0d1e7105..04d35c15d 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java @@ -4,19 +4,14 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -26,12 +21,10 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumPropertiesForSchemaOnly; @Testcontainers @@ -63,6 +56,16 @@ public void startContainers() throws InterruptedException { Thread.sleep(15000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + } + @Test public void testClickHouseDelayedStart() throws Exception { clickHouseContainer.getDockerClient().pauseContainerCmd(clickHouseContainer.getContainerId()).exec(); @@ -95,15 +98,9 @@ public void testClickHouseDelayedStart() throws Exception { while (true) { // Check if Batch was inserted. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); try { - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME where Type = 'DATETIME-INSERT55'"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME where Type = 'DATETIME-INSERT55'"); while (dateTimeResult.next()) { insertCheck = true; Assert.assertTrue(dateTimeResult.getString("Type").equalsIgnoreCase("DATETIME-INSERT55")); @@ -145,15 +142,7 @@ public void debeziumStorageView() throws Exception { Thread.sleep(10000); - // Connect to clickhouse and validate that the view was created successfully. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "altinity_sink_connector"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "altinity_sink_connector", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - - + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); // Check if the view altinity_sink_connector.show_replica_status was created successfully. ResultSet resultSet = writer.executeQueryWithResultSet("show create view altinity_sink_connector.show_replica_status"); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java index 5ab95eb1b..bcdbd133e 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java @@ -4,14 +4,13 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -24,9 +23,7 @@ import org.testcontainers.utility.DockerImageName; import java.sql.Connection; -import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,6 +62,11 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach() + public void stop() { + mySqlContainer.stop(); + clickHouseContainer.stop(); + } @DisplayName("Test that validates overriding database name in ClickHouse") @Test @@ -117,12 +119,7 @@ public void testDatabaseOverride() throws Exception { Thread.sleep(10000); // Validate in Clickhouse the last record written is 29999 - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); long col2 = 0L; ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from employees2.newtable final where col1 = 'a'"); @@ -157,5 +154,8 @@ public void testDatabaseOverride() throws Exception { conn.close(); // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideInitialIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideInitialIT.java index 465268279..40c922797 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideInitialIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideInitialIT.java @@ -4,17 +4,17 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.clickhouse.ClickHouseContainer; @@ -23,9 +23,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.utility.DockerImageName; -import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,6 +63,17 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } + @DisplayName("Test that validates overriding database name in ClickHouse") @Test @@ -99,13 +108,7 @@ public void testDatabaseOverride() throws Exception { Thread.sleep(10000); - // Validate in Clickhouse the last record written is 29999 - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); long col2 = 0L; ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from employees2.newtable final where col1 = 'a'"); @@ -136,5 +139,7 @@ public void testDatabaseOverride() throws Exception { // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideRRMTIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideRRMTIT.java index 182429f34..bdda2f8d0 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideRRMTIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideRRMTIT.java @@ -5,10 +5,9 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; @@ -23,19 +22,16 @@ import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.utility.DockerImageName; import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; import static org.junit.Assert.assertTrue; - public class DatabaseOverrideRRMTIT { private static final Logger log = LoggerFactory.getLogger(DatabaseOverrideRRMTIT.class); @@ -68,31 +64,26 @@ public void startContainers() throws InterruptedException { .withPassword("password") .withClasspathResourceMapping("config_replicated.xml", "/etc/clickhouse-server/config.d/config.xml", BindMode.READ_ONLY) .withClasspathResourceMapping("macros.xml", "/etc/clickhouse-server/config.d/macros.xml", BindMode.READ_ONLY) - .withExposedPorts(8123) - .waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); + .withExposedPorts(8123); + //.waitingFor(new HttpWaitStrategy().forPort(zookeeperContainer.getFirstMappedPort())); clickHouseContainer.withNetwork(network).withNetworkAliases("clickhouse"); - clickHouseContainer.start(); + //clickHouseContainer.start(); BasicConfigurator.configure(); mySqlContainer.start(); - clickHouseContainer.start(); + clickHouseContainer.start(); Thread.sleep(35000); - } + } @DisplayName("Test that validates overriding database name in ClickHouse for ReplicatedReplacingMergeTree(RRMT)") @Test public void testDatabaseOverride() throws Exception { - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - writer.executeQuery("CREATE DATABASE employees2"); - writer.executeQuery("CREATE DATABASE productsnew"); + writer.executeSystemQuery("CREATE DATABASE employees2"); + writer.executeSystemQuery("CREATE DATABASE productsnew"); Thread.sleep(10000); Injector injector = Guice.createInjector(new AppInjector()); @@ -211,5 +202,8 @@ public void testDatabaseOverride() throws Exception { conn.close(); // Files.deleteIfExists( tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java index 0fffe2464..fd568b020 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java @@ -2,20 +2,17 @@ import com.altinity.clickhouse.debezium.embedded.AppInjector; import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; +import org.junit.runner.RunWith; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -24,7 +21,6 @@ import java.sql.ResultSet; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -58,6 +54,17 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } + @Test @Disabled @DisplayName("Test that validates skipping of large tables in schema_only mode") @@ -89,14 +96,7 @@ public void testLoadingTablesInSchemaOnlyMode() throws Exception { Thread.sleep(25000); - // Confirm if only the whitelisted tabes were replicated. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); ResultSet dateTimeResult = writer.executeQueryWithResultSet("select name from system.tables where database='employees'"); boolean insertCheck = false; @@ -119,5 +119,8 @@ public void testLoadingTablesInSchemaOnlyMode() throws Exception { // Close connection. clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().stop(); + + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java index f3c8f2853..481cf339f 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java @@ -4,26 +4,18 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLBaseIT; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.model.DBCredentials; -import com.clickhouse.jdbc.ClickHouseConnection; import com.google.common.collect.Maps; import com.google.inject.Guice; import com.google.inject.Injector; import io.debezium.storage.jdbc.offset.JdbcOffsetBackingStoreConfig; - -import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; -import static org.junit.Assert.assertTrue; - import org.apache.log4j.BasicConfigurator; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; +import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.clickhouse.ClickHouseContainer; @@ -33,52 +25,21 @@ import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; -import java.sql.*; -import java.util.HashMap; +import java.sql.Connection; +import java.sql.ResultSet; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; +import static org.junit.Assert.assertTrue; + @Testcontainers public class DebeziumChangeEventCaptureIT{ private static final Logger log = LoggerFactory.getLogger(DebeziumChangeEventCaptureIT.class); - @Test - public void testDeleteOffsetStorageRow2() { - //System.out.println("Delete offset"); - DebeziumChangeEventCapture dec = new DebeziumChangeEventCapture(); - - try { - Properties props = getDebeziumProperties(mySqlContainer, clickHouseContainer); - props.setProperty("name", "altinity_sink_connector"); - Map propertiesMap = Maps.newHashMap(Maps.fromProperties(props)); - ClickHouseSinkConnectorConfig config = new ClickHouseSinkConnectorConfig(propertiesMap); - String tableName = props.getProperty(JdbcOffsetBackingStoreConfig.OFFSET_STORAGE_PREFIX + - JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); - DBCredentials dbCredentials = dec.parseDBConfiguration(config); - - String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase()); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "Client_1", dbCredentials.getUserName(), - dbCredentials.getPassword(), config); - - BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase(), dbCredentials.getUserName(), - dbCredentials.getPassword(), config, connection); - String offsetValue = new DebeziumOffsetStorage().getDebeziumStorageStatusQuery(props, writer); - //String offsetKey = new DebeziumOffsetStorage().getOffsetKey(props); - - String updateOffsetValue = new DebeziumOffsetStorage().updateBinLogInformation(offsetValue, "mysql-bin.001", "2333", null); - - //new DebeziumOffsetStorage().deleteOffsetStorageRow(offsetKey, props, writer); - //new DebeziumOffsetStorage().updateDebeziumStorageRow(writer, tableName, offsetKey, updateOffsetValue, System.currentTimeMillis()); - - System.out.print("Test"); - } catch(Exception e) { - log.error("Exception in testDeleteOffsetStorageRow2", e); - } - } + protected MySQLContainer mySqlContainer; @@ -105,6 +66,13 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach() + public void stop() { + mySqlContainer.stop(); + clickHouseContainer.stop(); + } + + @Test @DisplayName("Test that validates that the sequence number that is created in non-gtid mode is incremented correctly.") public void testIncrementingSequenceNumbers() throws Exception { @@ -149,36 +117,29 @@ public void testIncrementingSequenceNumbers() throws Exception { Thread.sleep(20000); - // Create connection to ClickHouse and get the version numbers. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); long version1 = 1L; long version2 = 1L; long version3 = 1L; long version4 = 1L; - ResultSet version1Result = writer.executeQueryWithResultSet("select _version from newtable final where col1 = 'a'"); + ResultSet version1Result = writer.executeQueryWithResultSet("select _version from employees.newtable final where col1 = 'a'"); while(version1Result.next()) { version1 = version1Result.getLong("_version"); } - ResultSet version2Result = writer.executeQueryWithResultSet("select _version from newtable final where col1 = 'b'"); + ResultSet version2Result = writer.executeQueryWithResultSet("select _version from employees.newtable final where col1 = 'b'"); while(version2Result.next()) { version2 = version2Result.getLong("_version"); } - ResultSet version3Result = writer.executeQueryWithResultSet("select _version from newtable final where col1 = 'c'"); + ResultSet version3Result = writer.executeQueryWithResultSet("select _version from employees.newtable final where col1 = 'c'"); while(version3Result.next()) { version3 = version3Result.getLong("_version"); } - ResultSet version4Result = writer.executeQueryWithResultSet("select _version from newtable final where col1 = 'd'"); + ResultSet version4Result = writer.executeQueryWithResultSet("select _version from employees.newtable final where col1 = 'd'"); while(version4Result.next()) { version4 = version4Result.getLong("_version"); } @@ -198,6 +159,9 @@ public void testIncrementingSequenceNumbers() throws Exception { clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().engine.close(); conn.close(); executorService.shutdown(); + + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureTest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureTest.java index 1c26bce8f..9c346b006 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureTest.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureTest.java @@ -6,15 +6,14 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; import org.json.simple.parser.ParseException; -import static org.junit.Assert.assertTrue; - -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; +import static org.junit.Assert.assertTrue; + public class DebeziumChangeEventCaptureTest { @Test diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java index dc5066fb5..bee63c5cc 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java @@ -2,21 +2,20 @@ import com.altinity.clickhouse.debezium.embedded.AppInjector; import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; -import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -61,6 +60,18 @@ public void startContainers() throws InterruptedException { Thread.sleep(15000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } + + @Test @DisplayName("Validates that the debezium storage view is created successfully") public void debeziumStorageView() throws Exception { @@ -86,8 +97,9 @@ public void debeziumStorageView() throws Exception { // Connect to clickhouse and validate that the view was created successfully. String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "altinity_sink_connector"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection chConn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "altinity_sink_connector", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); @@ -106,5 +118,7 @@ public void debeziumStorageView() throws Exception { ClickHouseDebeziumEmbeddedApplication.stop(); clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().stop(); executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java index 87d347f42..e7b64d582 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java @@ -4,18 +4,13 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.clickhouse.ClickHouseContainer; @@ -33,7 +28,6 @@ import java.util.concurrent.Executors; import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; -import static org.junit.Assert.assertTrue; @Testcontainers @Disabled @@ -67,6 +61,16 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } @Test public void testColumnMismatch() throws Exception { @@ -107,8 +111,9 @@ public void testColumnMismatch() throws Exception { // Validate in Clickhouse the last record written is 29999 String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection chConn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), + "employees", new ClickHouseSinkConnectorConfig(new HashMap<>())); BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java index f2a47165e..e7279a20a 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java @@ -4,17 +4,13 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.altinity.clickhouse.sink.connector.model.DBCredentials; -import com.clickhouse.jdbc.ClickHouseConnection; -import com.google.common.collect.Maps; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; -import io.debezium.storage.jdbc.offset.JdbcOffsetBackingStoreConfig; import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -30,8 +26,6 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -72,6 +66,16 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } @DisplayName("Test that validates that the sequence number that is created in non-gtid mode is incremented correctly," + "by performing a lot of updates on the primary key.") @@ -127,20 +131,14 @@ public void testIncrementingSequenceNumberWithUpdates() throws Exception { Thread.sleep(10000); - // Validate in Clickhouse the last record written is 29999 - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); long col2 = 1L; - ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from newtable final where col1 = 'a'"); + ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from employees.newtable final where col1 = 'a'"); while(version1Result.next()) { col2 = version1Result.getLong("col2"); } - Thread.sleep(10000); + Thread.sleep(15000); assertTrue(col2 == 29999); @@ -149,5 +147,8 @@ public void testIncrementingSequenceNumberWithUpdates() throws Exception { conn.close(); // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java index 98234cd8a..f2e9680a7 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java @@ -4,14 +4,13 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -65,6 +64,16 @@ public void startContainers() throws InterruptedException { Thread.sleep(35000); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } @Test public void testColumnMismatch() throws Exception { @@ -103,16 +112,10 @@ public void testColumnMismatch() throws Exception { Thread.sleep(10000); - // Validate in Clickhouse the last record written is 29999 - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); long col2 = 1L; - ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from newtable final where col1 = 'a'"); + ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from employees.newtable final where col1 = 'a'"); while(version1Result.next()) { col2 = version1Result.getLong("col2"); } @@ -123,6 +126,9 @@ public void testColumnMismatch() throws Exception { conn.close(); executorService.shutdown(); + + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java index 207340903..a27b58cf6 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java @@ -4,11 +4,9 @@ import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; -import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.classic.methods.HttpGet; import com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.classic.methods.HttpUriRequest; import com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -98,13 +96,7 @@ public void testRestClient() throws Exception { Thread.sleep(10000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); @@ -147,5 +139,7 @@ public void testRestClient() throws Exception { clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().stop(); + HikariDbSource.close(); + } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/AppConfigurationTest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/AppConfigurationTest.java deleted file mode 100644 index 2dbe955da..000000000 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/AppConfigurationTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.altinity.clickhouse.debezium.embedded.config; - - -import org.junit.Assert; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.SetEnvironmentVariable; - -import java.util.Properties; - -public class AppConfigurationTest { - - @Test - @Disabled - @SetEnvironmentVariable(key = "database.hostname", value = "localhost") - @SetEnvironmentVariable(key = "database.port", value = "8123") - @SetEnvironmentVariable(key = "database.user", value = "root") - @SetEnvironmentVariable(key = "database.pass", value = "password") - @SetEnvironmentVariable(key = "clickhouse.server.url", value = "localhost") - @SetEnvironmentVariable(key = "clickhouse.server.user", value = "root") - @SetEnvironmentVariable(key = "clickhouse.server.password", value = "root") - @SetEnvironmentVariable(key = "clickhouse.server.port", value = "8123") - @SetEnvironmentVariable(key = "snapshot.mode", value = "schema_only") - public void testParseConfiguration() throws Exception { - - EnvironmentConfigurationService environmentConfigurationService = new EnvironmentConfigurationService(); - Properties resultProperties = environmentConfigurationService.parse(); - - Assert.assertTrue(resultProperties.size() != 0); - - Assert.assertTrue(resultProperties.getProperty("database.hostname").equalsIgnoreCase("localhost")); - Assert.assertTrue(resultProperties.getProperty("database.port").equalsIgnoreCase("8123")); - Assert.assertTrue(resultProperties.getProperty("clickhouse.server.url").equalsIgnoreCase("localhost")); - - Assert.assertTrue(resultProperties.getProperty("snapshot.mode").equalsIgnoreCase("schema_only")); - } -} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/db/batch/BatchMultipleThreadsProcessingTest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/db/batch/BatchMultipleThreadsProcessingTest.java deleted file mode 100644 index 22f943402..000000000 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/db/batch/BatchMultipleThreadsProcessingTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.altinity.clickhouse.debezium.embedded.db.batch; - -import com.altinity.clickhouse.sink.connector.converters.ClickHouseConverter; -import com.altinity.clickhouse.sink.connector.executor.ClickHouseBatchRunnable; -import com.altinity.clickhouse.sink.connector.model.ClickHouseStruct; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.data.Struct; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - - -public class BatchMultipleThreadsProcessingTest { - - - @Test - @DisplayName("Integration test to test the batch multiple threads processing.") - public void testBatchMultipleThreadsProcessing() { - // Create the concurrentLinkedQueue of ClickHouseStruct - ClickHouseStruct ch1 = new ClickHouseStruct(10, "topic_1", getKafkaStruct(), 2, System.currentTimeMillis(), null, getKafkaStruct(), null, - ClickHouseConverter.CDC_OPERATION.CREATE); - // Add the ClickHouseStruct to the concurrentLinkedQueue - - // Create the ClickHouseBatchRunnable object - //ClickHouseBatchRunnable clickHouseBatchRunnable = new ClickHouseBatchRunnable(); - // Set the concurrentLinkedQueue to the ClickHouseBatchRunnable object - - } - - public static Struct getKafkaStruct() { - Schema kafkaConnectSchema = SchemaBuilder - .struct() - .field("first_name", Schema.STRING_SCHEMA) - .field("last_name", Schema.STRING_SCHEMA) - .field("quantity", Schema.INT32_SCHEMA) - .field("amount", Schema.FLOAT64_SCHEMA) - .field("employed", Schema.BOOLEAN_SCHEMA) - .build(); - - Struct kafkaConnectStruct = new Struct(kafkaConnectSchema); - kafkaConnectStruct.put("first_name", "John"); - kafkaConnectStruct.put("last_name", "Doe"); - kafkaConnectStruct.put("quantity", 100); - kafkaConnectStruct.put("amount", 23.223); - kafkaConnectStruct.put("employed", true); - - - return kafkaConnectStruct; - } -} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java index 66d77a177..20393c167 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java @@ -1,12 +1,11 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; + import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -18,7 +17,6 @@ import org.testcontainers.utility.DockerImageName; import java.sql.Connection; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -87,12 +85,8 @@ public void testAddColumn() throws Exception { Thread.sleep(25000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); - + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + Map shipClassColumns = writer.getColumnsDataTypesForTable("ship_class"); Map addTestColumns = writer.getColumnsDataTypesForTable("add_test"); @@ -117,6 +111,6 @@ public void testAddColumn() throws Exception { executorService.shutdown(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java index 33bbf1740..a2309acfc 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java @@ -1,10 +1,10 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -72,11 +72,7 @@ public void testChangeColumn() throws Exception { Thread.sleep(10000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); Map shipClassColumns = writer.getColumnsDataTypesForTable("ship_class"); Map addTestColumns = writer.getColumnsDataTypesForTable("add_test"); @@ -98,6 +94,6 @@ public void testChangeColumn() throws Exception { } executorService.shutdown(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java index 746139f18..83349d5b8 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java @@ -1,10 +1,10 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -68,10 +68,7 @@ public void testModifyColumn() throws Exception { Thread.sleep(15000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); Map shipClassColumns = writer.getColumnsDataTypesForTable("ship_class"); Map addTestColumns = writer.getColumnsDataTypesForTable("add_test"); @@ -92,6 +89,6 @@ public void testModifyColumn() throws Exception { } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableSchemaEvolutionIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableSchemaEvolutionIT.java deleted file mode 100644 index 9e8c8f939..000000000 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableSchemaEvolutionIT.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.altinity.clickhouse.debezium.embedded.ddl.parser; - -import org.junit.jupiter.api.DisplayName; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Testcontainers -@DisplayName("Integration Test that validates DDL replication of ALTER table column, first, after and MODIFY column") -public class AlterTableSchemaEvolutionIT extends DDLBaseIT { - -} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java index 149930ee9..3658b5060 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java @@ -3,12 +3,12 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -25,6 +25,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; +@Disabled @Testcontainers @DisplayName("Integration Test that validates auto create tables feature which creates tables when a CDC record(Insert) is received") public class AutoCreateTableIT { @@ -90,16 +91,9 @@ public void testAutoCreateTable(String clickHouseServerVersion) throws Exception Thread.sleep(10000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); Thread.sleep(10000); - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select count(*) from `new-table`"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select count(*) from employees.`new-table`"); boolean resultReceived = false; while(dateTimeResult.next()) { @@ -113,6 +107,6 @@ public void testAutoCreateTable(String clickHouseServerVersion) throws Exception } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java index 4eceaed79..a9e58ae5c 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java @@ -5,6 +5,7 @@ import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.metadata.DataTypeRange; import com.clickhouse.jdbc.ClickHouseConnection; import org.apache.log4j.BasicConfigurator; @@ -17,6 +18,7 @@ import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; +import java.sql.Connection; import java.sql.ResultSet; import java.util.HashMap; import java.util.Map; @@ -65,14 +67,7 @@ public void testCreateTable() throws Exception { Thread.sleep(40000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, - chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); Map decimalTable = writer.getColumnsDataTypesForTable("numeric_types_DECIMAL_65_30"); Map dateTimeTable6 = writer.getColumnsDataTypesForTable("temporal_types_DATETIME6"); @@ -110,13 +105,9 @@ public void testCreateTable() throws Exception { writer.getConnection().close(); //Thread.sleep(10000); - //String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + writer = ITCommon.getDBWriter(clickHouseContainer); // Validate temporal_types_DATE data. - ResultSet dateResult = writer.executeQueryWithResultSet("select * from temporal_types_DATE"); + ResultSet dateResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATE"); while(dateResult.next()) { Assert.assertTrue(dateResult.getDate("Minimum_Value").toString().equalsIgnoreCase("1900-01-01")); @@ -124,7 +115,7 @@ public void testCreateTable() throws Exception { Assert.assertTrue(dateResult.getDate("Maximum_Value").toString().equalsIgnoreCase("2299-12-31")); } // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); /** DATE TIME @@ -173,7 +164,7 @@ public void testCreateTable() throws Exception { } // DATETIME1 - ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME1"); + ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME1"); while(dateTimeResult1.next()) { System.out.println("DATE TIME 1"); @@ -188,7 +179,7 @@ public void testCreateTable() throws Exception { } // DATETIME2 - ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2"); + ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME2"); while(dateTimeResult2.next()) { System.out.println("DATE TIME 2"); @@ -314,5 +305,7 @@ public void testCreateTable() throws Exception { executorService.shutdown(); writer.getConnection().close(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java index aa7418596..001dcc8c8 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java @@ -3,9 +3,8 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -19,6 +18,7 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; +import java.sql.Connection; import java.sql.ResultSet; import java.util.HashMap; import java.util.Map; @@ -78,14 +78,7 @@ public void testCreateTable() throws Exception { Thread.sleep(30000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); Map decimalTable = writer.getColumnsDataTypesForTable("numeric_types_DECIMAL_65_30"); Map dateTimeTable = writer.getColumnsDataTypesForTable("temporal_types_DATETIME6"); @@ -112,16 +105,11 @@ public void testCreateTable() throws Exception { Assert.assertTrue(timestampTable.get("Maximum_Value").equalsIgnoreCase("DateTime64(6)")); Assert.assertTrue(timestampTable.get("Null_Value").equalsIgnoreCase("Nullable(DateTime64(6))")); - writer.getConnection().close(); + //writer.getConnection().close(); Thread.sleep(10000); - ClickHouseConnection conn2 = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn2); // Validate temporal_types_DATE data. - ResultSet dateResult = writer.executeQueryWithResultSet("select * from temporal_types_DATE"); + ResultSet dateResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATE"); boolean dateResultValueChecked = false; while(dateResult.next()) { dateResultValueChecked = true; @@ -138,7 +126,7 @@ public void testCreateTable() throws Exception { Assert.assertTrue(dateResultValueChecked); // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); boolean dateTimeResultValueChecked = false; @@ -196,7 +184,7 @@ public void testCreateTable() throws Exception { // DATETIME1 boolean dateTimeResult1ValueChecked = false; - ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME1"); + ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME1"); while(dateTimeResult1.next()) { System.out.println("DATE TIME 1"); dateTimeResult1ValueChecked = true; @@ -212,7 +200,7 @@ public void testCreateTable() throws Exception { Assert.assertTrue(dateTimeResult1ValueChecked); // DATETIME2 - ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2"); + ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME2"); while(dateTimeResult2.next()) { System.out.println("DATE TIME 2"); System.out.println(dateTimeResult2.getTimestamp("Mid_Value").toString()); @@ -293,6 +281,8 @@ public void testCreateTable() throws Exception { executorService.shutdown(); writer.getConnection().close(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java index 412c94828..dc1cb5cf9 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java @@ -3,6 +3,8 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; + + import org.apache.log4j.BasicConfigurator; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -14,8 +16,6 @@ import org.testcontainers.utility.DockerImageName; import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import java.util.Properties; @Testcontainers diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLIgnoreRegExIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLIgnoreRegExIT.java index 72673badd..efe926a8e 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLIgnoreRegExIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLIgnoreRegExIT.java @@ -13,11 +13,9 @@ import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; -import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.junit.Assert; -import java.util.HashMap; + import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; @@ -88,13 +86,7 @@ public void testDDLIgnoreRegex() throws Exception { Thread.sleep(10000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); - - - // Thread.sleep(5000); + // Thread.sleep(5000); // Run MySQL DDL to run analyze partition. String analyzePartitionDDL = "alter table sales analyze partition p2022"; ITCommon.connectToMySQL(mySqlContainer).createStatement().executeUpdate(analyzePartitionDDL); @@ -130,6 +122,7 @@ public void testDDLIgnoreRegex() throws Exception { engine.get().stop(); } executorService.shutdown(); + HikariDbSource.close(); } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java index a9f3ac7e7..66496ae16 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java @@ -1,12 +1,13 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -127,15 +128,9 @@ public void testSchemaOnlyMode() throws Exception { Thread.sleep(10000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); while(dateTimeResult.next()) { System.out.println("DATE TIME"); @@ -153,7 +148,7 @@ public void testSchemaOnlyMode() throws Exception { } // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); - + HikariDbSource.close(); } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java index 5c6ec0249..7e2c40008 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java @@ -2,14 +2,12 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; -import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import junit.framework.Assert; import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -21,6 +19,7 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; +import java.sql.Connection; import java.sql.ResultSet; import java.util.HashMap; import java.util.Properties; @@ -58,6 +57,12 @@ public void startContainers() throws InterruptedException { Thread.sleep(15000); } + @AfterEach + public void tearDown() { + mySqlContainer.stop(); + clickHouseContainer.stop(); + } + @Test public void testCreateTable() throws Exception { AtomicReference engine = new AtomicReference<>(); @@ -79,17 +84,10 @@ public void testCreateTable() throws Exception { Thread.sleep(30000); // Create connection. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); while(dateTimeResult.next()) { System.out.println("DATE TIME"); @@ -103,7 +101,7 @@ public void testCreateTable() throws Exception { } // DATETIME1 - ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME1"); + ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME1"); while(dateTimeResult1.next()) { System.out.println("DATE TIME 1"); System.out.println(dateTimeResult1.getTimestamp("Minimum_Value").toString()); @@ -116,7 +114,7 @@ public void testCreateTable() throws Exception { } // DATETIME2 - ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2"); + ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME2"); while(dateTimeResult2.next()) { System.out.println("DATE TIME 2"); System.out.println(dateTimeResult2.getTimestamp("Minimum_Value").toString()); @@ -197,6 +195,8 @@ public void testCreateTable() throws Exception { executorService.shutdown(); writer.getConnection().close(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java index 27230aa48..7571057ca 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java @@ -1,12 +1,12 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -94,19 +94,11 @@ public void testCreateTable() throws Exception { //conn.prepareStatement("INSERT INTO `temporal_types_DATETIME` VALUES ('DATETIME-INSERT','1000-01-01 00:00:00','2022-09-29 01:47:46','9999-12-31 23:59:59',NULL);\n").execute(); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection conn1 = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, - conn1); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); - writer.getConnection().close(); - Thread.sleep(10000); - ClickHouseConnection conn2 = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn2); + //writer.getConnection().close(); + Thread.sleep(20000); /** * DATE TIME @@ -123,7 +115,7 @@ public void testCreateTable() throws Exception { * 2106-02-07 00:28:15.0 */ // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); while(dateTimeResult.next()) { System.out.println("DATE TIME"); @@ -137,7 +129,7 @@ public void testCreateTable() throws Exception { } // DATETIME1 - ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME1"); + ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME1"); while(dateTimeResult1.next()) { System.out.println("DATE TIME 1"); System.out.println(dateTimeResult1.getTimestamp("Minimum_Value").toString()); @@ -150,7 +142,7 @@ public void testCreateTable() throws Exception { } // DATETIME2 - ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2"); + ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME2"); while(dateTimeResult2.next()) { System.out.println("DATE TIME 2"); System.out.println(dateTimeResult2.getTimestamp("Minimum_Value").toString()); @@ -245,6 +237,7 @@ public void testCreateTable() throws Exception { executorService.shutdown(); writer.getConnection().close(); + HikariDbSource.close(); } protected Properties getDebeziumProperties() throws Exception { diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java index 887b04d62..a40fe575a 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java @@ -1,12 +1,12 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -107,13 +107,7 @@ public void testCreateTable() throws Exception { String DATETIME6_MID = "2022-09-29 01:50:56.123456"; String DATETIME6_MAX = "2299-12-31 23:59:59.999999"; - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); // Validate that the MySQL server is set to Central timezone. @@ -152,7 +146,7 @@ public void testCreateTable() throws Exception { * 2106-02-07 00:28:15.0 */ // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); while(dateTimeResult.next()) { System.out.println("DATE TIME"); @@ -167,7 +161,7 @@ public void testCreateTable() throws Exception { } // DATETIME1 - ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME1"); + ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME1"); while(dateTimeResult1.next()) { System.out.println("DATE TIME 1"); System.out.println(dateTimeResult1.getTimestamp("Minimum_Value").toString()); @@ -180,7 +174,7 @@ public void testCreateTable() throws Exception { } // DATETIME2 - ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2"); + ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME2"); while(dateTimeResult2.next()) { System.out.println("DATE TIME 2"); System.out.println(dateTimeResult2.getTimestamp("Minimum_Value").toString()); @@ -260,6 +254,8 @@ public void testCreateTable() throws Exception { executorService.shutdown(); writer.getConnection().close(); + + HikariDbSource.close(); } protected Properties getDebeziumProperties() throws Exception { diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java index 0920e706a..09a2a6bb2 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java @@ -1,13 +1,12 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.altinity.clickhouse.sink.connector.metadata.DataTypeRange; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -25,7 +24,6 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -94,24 +92,12 @@ public void testCreateTable() throws Exception { conn.prepareStatement("INSERT INTO `temporal_types_DATETIME6` VALUES ('DATETIME(6)-INSERT','1000-01-01 00:00:00.000000','2022-09-29 01:50:56.123456','9999-12-31 23:59:59.999999',NULL)").execute(); //conn.prepareStatement("INSERT INTO `temporal_types_DATETIME` VALUES ('DATETIME-INSERT','1000-01-01 00:00:00','2022-09-29 01:47:46','9999-12-31 23:59:59',NULL);\n").execute(); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); writer.getConnection().close(); Thread.sleep(10000); - ClickHouseConnection chConn2 = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn2); - + writer = ITCommon.getDBWriter(clickHouseContainer); /** * DATE TIME * 1969-12-31 18:00:00.0 @@ -127,7 +113,7 @@ public void testCreateTable() throws Exception { * 2106-02-07 00:28:15.0 */ // Validate temporal_types_DATETIME data. - ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME"); + ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME"); boolean dateTimeResultChecked = false; while(dateTimeResult.next()) { @@ -144,7 +130,7 @@ public void testCreateTable() throws Exception { Assert.assertTrue(dateTimeResultChecked); // DATETIME1 - ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME1"); + ResultSet dateTimeResult1 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME1"); while(dateTimeResult1.next()) { System.out.println("DATE TIME 1"); System.out.println(dateTimeResult1.getTimestamp("Minimum_Value").toString()); @@ -157,7 +143,7 @@ public void testCreateTable() throws Exception { } // DATETIME2 - ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2"); + ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME2"); while(dateTimeResult2.next()) { System.out.println("DATE TIME 2"); System.out.println(dateTimeResult2.getTimestamp("Minimum_Value").toString()); @@ -237,6 +223,8 @@ public void testCreateTable() throws Exception { executorService.shutdown(); writer.getConnection().close(); + + HikariDbSource.close(); } protected Properties getDebeziumProperties() throws Exception { diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java index e5d8d654e..47c3662f6 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java @@ -1,10 +1,10 @@ package com.altinity.clickhouse.debezium.embedded.ddl.parser; +import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -18,7 +18,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -76,10 +75,7 @@ public void testEmployeesDB() throws Exception { Thread.sleep(40000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), "employees"); - ClickHouseConnection connection = BaseDbWriter.createConnection(jdbcUrl, "client_1", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, connection); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); // Validate that all the tables are created. @@ -136,6 +132,7 @@ public void testEmployeesDB() throws Exception { } executorService.shutdown(); + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java index 28382c53b..cdc00f42d 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java @@ -3,9 +3,8 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -20,7 +19,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; @@ -92,14 +90,8 @@ public void testIsDeleted(String clickHouseServerVersion) throws Exception { conn.close(); Thread.sleep(10000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); - ResultSet rs = writer.executeQueryWithResultSet("select * from new_table"); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); + ResultSet rs = writer.executeQueryWithResultSet("select * from employees.new_table"); boolean recordFound = false; while(rs.next()) { recordFound = true; @@ -114,5 +106,7 @@ public void testIsDeleted(String clickHouseServerVersion) throws Exception { engine.get().stop(); } executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java index 07db78f4f..2cf4c7cc3 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java @@ -3,14 +3,11 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -19,7 +16,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -62,6 +58,16 @@ public void startContainers() throws InterruptedException { clickHouseContainer.start(); } + @AfterEach + public void stopContainers() { + if(mySqlContainer != null && mySqlContainer.isRunning()) { + mySqlContainer.stop();; + } + if(clickHouseContainer != null && clickHouseContainer.isRunning()) { + clickHouseContainer.stop(); + } + + } @DisplayName("Integration Test that validates handling of multiple databases") @Test public void testMultipleDatabases() throws Exception { @@ -119,13 +125,7 @@ public void testMultipleDatabases() throws Exception { conn.close(); // Create connection to clickhouse and validate if the tables are replicated. - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "system", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer, "test_db"); // query clickhouse connection and get data for test_table1 and test_table2 @@ -160,12 +160,7 @@ public void testMultipleDatabases() throws Exception { // Create a test_db DBWriter instance. // A new ClickHouseConnection with test_db database. // Jdbc url with test_db database. - String testDb2JdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "test_db2"); - ClickHouseConnection testDb2Conn = BaseDbWriter.createConnection(testDb2JdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - BaseDbWriter testDb2Writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "test_db2", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, testDb2Conn); + BaseDbWriter testDb2Writer = ITCommon.getDBWriter(clickHouseContainer, "test_db2"); // Validate the columns in Clickhouse for test_db.test_table Map columnMap = testDb2Writer.getColumnsDataTypesForTable("test_table"); @@ -181,6 +176,6 @@ public void testMultipleDatabases() throws Exception { writer.getConnection().close(); - + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java index 2678f81fe..170582a58 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java @@ -5,15 +5,14 @@ import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; @@ -122,13 +121,7 @@ public void testTableOperations(boolean databaseOverride) throws Exception { "fullname varchar(101) GENERATED ALWAYS AS (CONCAT(first_name,' ',last_name)),\n" + "email VARCHAR(100) NOT NULL);\n").execute(); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); conn.prepareStatement("create table new_table_copy like new_table").execute(); @@ -142,7 +135,7 @@ public void testTableOperations(boolean databaseOverride) throws Exception { // Validate table created with partitions. - String membersResult = writer.executeQuery("show create table members"); + String membersResult = writer.executeSystemQuery("show create table members"); Assert.assertTrue(membersResult.equalsIgnoreCase("CREATE TABLE employees.members\n" + "(\n" + " `firstname` String,\n" + @@ -158,7 +151,7 @@ public void testTableOperations(boolean databaseOverride) throws Exception { "ORDER BY tuple()\n" + "SETTINGS index_granularity = 8192")); - String rcxResult = writer.executeQuery("show create table rcx"); + String rcxResult = writer.executeSystemQuery("show create table rcx"); Assert.assertTrue(rcxResult.equalsIgnoreCase("CREATE TABLE employees.rcx\n" + "(\n" + @@ -184,6 +177,7 @@ public void testTableOperations(boolean databaseOverride) throws Exception { // Files.deleteIfExists(tmpFilePath); executorService.shutdown(); + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java index 62518cabe..5d2341177 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java @@ -3,11 +3,11 @@ import com.altinity.clickhouse.debezium.embedded.ITCommon; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; -import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.HikariDbSource; import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.clickhouse.jdbc.ClickHouseConnection; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -59,6 +59,11 @@ public void startContainers() throws InterruptedException { clickHouseContainer.start(); } + @AfterEach + public void stop() { + mySqlContainer.stop(); + clickHouseContainer.stop(); + } @Test @DisplayName("Test that validates create table in CH when MySQL has is_deleted columns") @@ -79,18 +84,12 @@ public void testIsDeleted() throws Exception { Thread.sleep(30000); - String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees"); - ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); - - BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), - "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + BaseDbWriter writer = ITCommon.getDBWriter(clickHouseContainer); //Validate if ship_class was truncated also in ClickHouse. // Validate that the table is empty in ClickHouse - ResultSet rs = writer.executeQueryWithResultSet("select * from ship_class"); + ResultSet rs = writer.executeQueryWithResultSet("select * from employees.ship_class"); boolean recordFoundShipClass = false; while(rs.next()) { recordFoundShipClass = true; @@ -106,7 +105,7 @@ public void testIsDeleted() throws Exception { conn.close(); Thread.sleep(10000); - rs = writer.executeQueryWithResultSet("select * from new_table"); + rs = writer.executeQueryWithResultSet("select * from employees.new_table"); boolean recordFound = false; while(rs.next()) { recordFound = true; @@ -124,7 +123,7 @@ public void testIsDeleted() throws Exception { conn.close(); // Validate that the table is empty in ClickHouse - rs = writer.executeQueryWithResultSet("select * from new_table"); + rs = writer.executeQueryWithResultSet("select * from employees.new_table"); recordFound = false; while(rs.next()) { recordFound = true; @@ -136,5 +135,7 @@ public void testIsDeleted() throws Exception { engine.get().stop(); } executorService.shutdown(); + + HikariDbSource.close(); } } diff --git a/sink-connector-lightweight/tests/integration/helpers/default_config.py b/sink-connector-lightweight/tests/integration/helpers/default_config.py index af92e9329..dc424c68c 100644 --- a/sink-connector-lightweight/tests/integration/helpers/default_config.py +++ b/sink-connector-lightweight/tests/integration/helpers/default_config.py @@ -48,4 +48,5 @@ "clickhouse.datetime.timezone": "UTC", "auto.create.tables": "true", "ddl.retry": "true", + "connection.pool.max.size": "3000" } diff --git a/sink-connector/docker/grafana/config/altinity_sink_connector.json b/sink-connector/docker/grafana/config/altinity_sink_connector.json index 8b4862634..37c47108d 100644 --- a/sink-connector/docker/grafana/config/altinity_sink_connector.json +++ b/sink-connector/docker/grafana/config/altinity_sink_connector.json @@ -24,17 +24,326 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 1, "links": [], "liveNow": false, "panels": [ + { + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "PDF61E9E97939C7ED" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 65, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "adHocFilters": [], + "adHocValuesQuery": "", + "add_metadata": false, + "contextWindowSize": "10", + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "PDF61E9E97939C7ED" + }, + "editorMode": "builder", + "extrapolate": false, + "format": "table", + "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", + "interval": "", + "intervalFactor": 1, + "query": "select seconds_behind_source from altinity_sink_connector.show_replica_status", + "rawQuery": "select seconds_behind_source from altinity_sink_connector.show_replica_status", + "refId": "A", + "round": "0s", + "showFormattedSQL": true, + "skip_comments": false, + "useWindowFuncForMacros": false + } + ], + "title": "Seconds Behind source", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "hikaricp_connections", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "hikaricp_connections_max", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Connection Pool(ClickHouse)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "topic" + }, + "properties": [ + { + "id": "custom.width", + "value": 324 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 109 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(clickhouse_sink_topics_num_records_total[$__rate_interval])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Records (Topic)", + "type": "table" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 0 + "y": 13 }, "id": 26, "panels": [], @@ -50,7 +359,7 @@ "h": 5, "w": 2, "x": 0, - "y": 1 + "y": 14 }, "id": 2, "options": { @@ -62,7 +371,7 @@ "content": "
\n \n

Altinity Sink Connector for ClickHouse

\n

For more information, visit ", "mode": "html" }, - "pluginVersion": "9.4.7", + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -116,7 +425,7 @@ "h": 5, "w": 3, "x": 2, - "y": 1 + "y": 14 }, "id": 4, "options": { @@ -124,6 +433,7 @@ "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -131,10 +441,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "value_and_name" + "textMode": "value_and_name", + "wideLayout": true }, - "pluginVersion": "9.4.7", + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -180,7 +492,7 @@ "h": 5, "w": 2, "x": 5, - "y": 1 + "y": 14 }, "id": 6, "options": { @@ -188,6 +500,7 @@ "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -195,10 +508,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.4.7", + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -247,7 +562,7 @@ "h": 5, "w": 2, "x": 7, - "y": 1 + "y": 14 }, "id": 50, "options": { @@ -255,6 +570,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -262,9 +578,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.4.7", + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -279,223 +597,17 @@ } ], "title": "Errors", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 60, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "expr": "clickhouse_sink_connector_uptime/1000", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "UpTime(secs)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "topic" - }, - "properties": [ - { - "id": "custom.width", - "value": 324 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.width", - "value": 72 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 109 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 8, - "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(clickhouse_sink_topics_num_records_total[$__rate_interval])", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Records (Topic)", - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": -1, - "drawStyle": "bars", - "fillOpacity": 97, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -515,24 +627,30 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 + "h": 5, + "w": 3, + "x": 9, + "y": 14 }, - "id": 16, + "id": 60, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -540,13 +658,14 @@ "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", - "expr": "clickhouse_sink_debezium_lag/1000", + "expr": "clickhouse_sink_connector_uptime/1000", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Debezium -> CH Lag (secs)", - "type": "timeseries" + "title": "UpTime(secs)", + "type": "stat" }, { "datasource": { @@ -560,31 +679,22 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, "thresholdsStyle": { "mode": "off" } @@ -610,20 +720,29 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 14 }, "id": 10, "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "targets": [ { @@ -640,45 +759,35 @@ } ], "title": "Rate(Total Records)", - "type": "timeseries" + "type": "barchart" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "description": "Lag calculated from time records are inserted to CH and the timestamp recorded by debezium connector", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, "thresholdsStyle": { "mode": "off" } @@ -704,36 +813,44 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 19 }, - "id": 14, + "id": 16, "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", - "expr": "clickhouse_sink_db_lag/1000", - "legendFormat": "__auto", + "expr": "clickhouse_sink_debezium_lag/1000", "range": true, "refId": "A" } ], - "title": "Source DB -> CH Lag (secs)", - "type": "timeseries" + "title": "Debezium -> CH Lag (secs)", + "type": "barchart" }, { "datasource": { @@ -774,10 +891,11 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 22 }, "id": 62, "options": { + "cellHeight": "sm", "footer": { "countRows": false, "fields": "", @@ -788,7 +906,7 @@ }, "showHeader": true }, - "pluginVersion": "9.4.7", + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -836,40 +954,32 @@ }, { "datasource": { - "type": "vertamedia-clickhouse-datasource", - "uid": "PDF61E9E97939C7ED" + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, + "description": "Lag calculated from time records are inserted to CH and the timestamp recorded by debezium connector", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 49, - "gradientMode": "opacity", + "fillOpacity": 80, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, "thresholdsStyle": { "mode": "off" } @@ -895,20 +1005,122 @@ "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 27 }, - "id": 64, + "id": 14, "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "clickhouse_sink_db_lag/1000", + "legendFormat": "__auto", + "range": true, + "refId": "A" } + ], + "title": "Source DB -> CH Lag (secs)", + "type": "barchart" + }, + { + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "PDF61E9E97939C7ED" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 64, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xField": "Time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": -200 }, "pluginVersion": "9.4.7", "targets": [ @@ -922,17 +1134,16 @@ "format": "time_series", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, - "interval": "", "intervalFactor": 1, - "query": "SELECT\n toUnixTimestamp(toStartOfMinute(event_time)) * 1000 AS t,\n sum(rows)/60 as rows,\n table\nFROM system.part_log\n\nWHERE\n (event_type = 'NewPart')\n AND event_time >= toDateTime($from)\n and event_time < toDateTime($to)\nGROUP BY\n t,\n table\nORDER BY\n t,\n table", - "rawQuery": "SELECT\n toUnixTimestamp(toStartOfMinute(event_time)) * 1000 AS t,\n sum(rows)/60 as rows,\n table\nFROM system.part_log\n\nWHERE\n (event_type = 'NewPart')\n AND event_time >= toDateTime(1684726913)\n and event_time < toDateTime(1684727213)\nGROUP BY\n t,\n table\nORDER BY\n t,\n table", + "query": "SELECT toStartOfInterval(event_time, toIntervalSecond(1)) AS _time_dec,\nsum(rows) as rows, table\nFROM system.part_log\nWHERE (event_type = 'NewPart') AND event_time >= toDateTime($from) and event_time < toDateTime($to)\nGROUP BY _time_dec, table", + "rawQuery": "SELECT toStartOfInterval(event_time, toIntervalSecond(1)) AS _time_dec,\nsum(rows) as rows, table\nFROM system.part_log\nWHERE (event_type = 'NewPart') AND event_time >= toDateTime(1681314113) and event_time < toDateTime(1681335713)\nGROUP BY _time_dec, table", "refId": "A", - "round": "1s", + "round": "0s", "skip_comments": true } ], "title": "Rows inserted ", - "type": "timeseries" + "type": "barchart" }, { "collapsed": false, @@ -940,7 +1151,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 43 }, "id": 28, "panels": [], @@ -978,7 +1189,7 @@ "h": 8, "w": 4, "x": 0, - "y": 31 + "y": 44 }, "id": 30, "options": { @@ -986,6 +1197,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -993,9 +1205,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.4.7", + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -1023,6 +1237,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1036,6 +1251,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1073,7 +1289,7 @@ "h": 8, "w": 10, "x": 4, - "y": 31 + "y": 44 }, "id": 32, "options": { @@ -1115,6 +1331,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1128,6 +1345,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1165,7 +1383,7 @@ "h": 8, "w": 10, "x": 14, - "y": 31 + "y": 44 }, "id": 34, "options": { @@ -1242,8 +1460,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1258,7 +1475,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 52 }, "id": 48, "options": { @@ -1295,7 +1512,7 @@ "h": 1, "w": 24, "x": 0, - "y": 47 + "y": 60 }, "id": 22, "panels": [], @@ -1317,27 +1534,17 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, "thresholdsStyle": { "mode": "off" } @@ -1347,8 +1554,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1363,20 +1569,29 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 61 }, "id": 24, "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "targets": [ { @@ -1392,7 +1607,7 @@ } ], "title": "Memory Used(MB)", - "type": "timeseries" + "type": "barchart" }, { "datasource": { @@ -1409,27 +1624,17 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, "thresholdsStyle": { "mode": "off" } @@ -1439,8 +1644,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1455,20 +1659,29 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 61 }, "id": 18, "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "targets": [ { @@ -1483,7 +1696,7 @@ } ], "title": "Memory Max (MB)", - "type": "timeseries" + "type": "barchart" }, { "collapsed": false, @@ -1491,7 +1704,7 @@ "h": 1, "w": 24, "x": 0, - "y": 56 + "y": 69 }, "id": 36, "panels": [], @@ -1513,8 +1726,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1529,7 +1741,7 @@ "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 70 }, "id": 38, "options": { @@ -1546,7 +1758,7 @@ }, "textMode": "auto" }, - "pluginVersion": "9.4.7", + "pluginVersion": "9.1.1", "targets": [ { "datasource": { @@ -1578,8 +1790,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1594,7 +1805,7 @@ "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 70 }, "id": 40, "options": { @@ -1611,7 +1822,7 @@ }, "textMode": "auto" }, - "pluginVersion": "9.4.7", + "pluginVersion": "9.1.1", "targets": [ { "datasource": { @@ -1634,7 +1845,7 @@ "h": 1, "w": 24, "x": 0, - "y": 65 + "y": 78 }, "id": 42, "panels": [], @@ -1656,8 +1867,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1672,7 +1882,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 79 }, "id": 44, "options": { @@ -1689,7 +1899,7 @@ }, "textMode": "auto" }, - "pluginVersion": "9.4.7", + "pluginVersion": "9.1.1", "targets": [ { "datasource": { @@ -1721,29 +1931,16 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } }, "mappings": [], @@ -1751,8 +1948,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1767,20 +1963,28 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 79 }, "id": 46, "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "targets": [ { @@ -1796,25 +2000,24 @@ } ], "title": "Live Thread States", - "type": "timeseries" + "type": "barchart" } ], - "refresh": "", + "refresh": false, "revision": 1, - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [] }, "time": { - "from": "now-15m", + "from": "now-5m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Altinity Sink Connector for ClickHouse", - "uid": "c4cKtgk4p", - "version": 2, + "uid": "c4cKtgk4k", + "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/sink-connector/pom.xml b/sink-connector/pom.xml index 597bb9865..146cb2413 100644 --- a/sink-connector/pom.xml +++ b/sink-connector/pom.xml @@ -313,7 +313,13 @@ debezium-core ${version.debezium} - + + io.debezium + debezium-storage-jdbc + ${version.debezium} + + org.projectlombok lombok @@ -333,6 +339,11 @@ maven-artifact 3.9.1 + + com.zaxxer + HikariCP + 6.0.0 + com.clickhouse diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java index 11a9a0bd7..b4e0989a9 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java @@ -474,7 +474,46 @@ static ConfigDef newConfigDef() { 6, ConfigDef.Width.NONE, ClickHouseSinkConnectorConfigVariables.REPLICA_STATUS_VIEW.toString()) - + .define( + ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_MAX_SIZE.toString(), + Type.INT, + 500, + Importance.HIGH, + "The maximum size of the connection pool", + CONFIG_GROUP_CONNECTOR_CONFIG, + 6, + ConfigDef.Width.NONE, + ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_MAX_SIZE.toString()) + .define( + ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_TIMEOUT.toString(), + Type.LONG, + 50000, + Importance.HIGH, + "The timeout for the connection pool", + CONFIG_GROUP_CONNECTOR_CONFIG, + 6, + ConfigDef.Width.NONE, + ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_TIMEOUT.toString()) + .define( + ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_MIN_IDLE.toString(), + Type.INT, + 10, + Importance.HIGH, + "The minimum number of idle connections in the connection pool", + CONFIG_GROUP_CONNECTOR_CONFIG, + 6, + ConfigDef.Width.NONE, + ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_MIN_IDLE.toString()) + .define( + ClickHouseSinkConnectorConfigVariables.OFFSET_STORAGE_TABLE_NAME.toString(), + Type.STRING, + "altinity_sink_connector", + Importance.HIGH, + "The name of the offset storage table", + CONFIG_GROUP_CONNECTOR_CONFIG, + 7, + ConfigDef.Width.NONE, + ClickHouseSinkConnectorConfigVariables.OFFSET_STORAGE_TABLE_NAME.toString()) ; } } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java index b4fddc77c..7632bc869 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java @@ -75,7 +75,13 @@ public enum ClickHouseSinkConnectorConfigVariables { REPLICA_STATUS_VIEW("replica.status.view"), MAX_QUEUE_SIZE("sink.connector.max.queue.size"), - SINGLE_THREADED("single.threaded"); + SINGLE_THREADED("single.threaded"), + + CONNECTION_POOL_MAX_SIZE("connection.pool.max.size"), + CONNECTION_POOL_TIMEOUT("connection.pool.timeout"), + CONNECTION_POOL_MIN_IDLE("connection.pool.min.idle"), + + OFFSET_STORAGE_TABLE_NAME("offset.storage.jdbc.offset.table.name"); private String label; diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Metrics.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Metrics.java index 5dd3da440..7c64de41e 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Metrics.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Metrics.java @@ -81,17 +81,6 @@ public static void initialize(String enableFlag, String metricsPort) { connectorStartTimeMs = System.currentTimeMillis(); - // Register reporters here. -// reporter = ConsoleReporter.forRegistry(registry) -// .convertRatesTo(TimeUnit.SECONDS) -// .convertDurationsTo(TimeUnit.SECONDS) -// .build(); -// reporter.start(1, TimeUnit.MINUTES); - - // registry = new MetricRegistry(); -// registry.register("memory", new MemoryUsageGaugeSet()); -// registry.register("jvm.thread-states",new ThreadStatesGaugeSet()); -// registry.register("jvm.garbage-collector",new GarbageCollectorMetricSet()); parseConfiguration(enableFlag, metricsPort); if(enableMetrics) { diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriter.java index 8043611a8..be65142f9 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriter.java @@ -2,9 +2,10 @@ import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; -import com.clickhouse.jdbc.ClickHouseConnection; import com.clickhouse.jdbc.ClickHouseDataSource; + +import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.time.ZoneId; @@ -12,12 +13,16 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; + +import com.zaxxer.hikari.HikariDataSource; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; public class BaseDbWriter { - protected ClickHouseConnection conn; + public static final String DATABASE_CLIENT_NAME = "Sink_Connector"; + public static final String SYSTEM_DB = "system"; + protected Connection conn; private String hostName; private Integer port; @@ -38,7 +43,7 @@ public BaseDbWriter( String userName, String password, ClickHouseSinkConnectorConfig config, - ClickHouseConnection conn + Connection conn ) { this.hostName = hostName; @@ -72,7 +77,8 @@ public static Properties splitJdbcProperties(String jdbcProperties) { return properties; } - public ClickHouseConnection getConnection() { + public Connection getConnection() { + HikariDbSource.printConnectionInfo(); return this.conn; } public static String getConnectionString(String hostName, Integer port, String database) { @@ -87,11 +93,12 @@ public static String getConnectionString(String hostName, Integer port, String d * @param userName UserName * @param password Password */ - public static ClickHouseConnection createConnection(String url, String clientName, String userName, String password, - ClickHouseSinkConnectorConfig config) { + public static Connection createConnection(String url, String clientName, String userName, + String password, String databaseName + , ClickHouseSinkConnectorConfig config) { String jdbcParams = ""; - ClickHouseConnection conn = null; + Connection conn = null; if(config != null) { config.getString(ClickHouseSinkConnectorConfigVariables.JDBC_PARAMETERS.toString()); } @@ -106,8 +113,16 @@ public static ClickHouseConnection createConnection(String url, String clientNam Properties userProps = splitJdbcProperties(jdbcParams); properties.putAll(userProps); } + // Add username/password to the url. + url = url + "?user=" + userName + "&password=" + password; ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties); - conn = dataSource.getConnection(userName, password); + // Get connection from the pool. + HikariDataSource hikariDbSource = HikariDbSource.getInstance(dataSource, databaseName, config); + // Create a new ClickHouseConnection object with the connection from the pool. + // Convert Connection to ClickHouseConnection. + + conn = hikariDbSource.getConnection(); + //conn = dataSource.getConnection(userName, password); } catch (Exception e) { log.error("Error creating ClickHouse connection" + e); } @@ -123,12 +138,12 @@ public static ClickHouseConnection createConnection(String url, String clientNam * @return * @throws SQLException */ - public String executeQuery(String sql) throws SQLException { + public String executeSystemQuery(String sql) throws SQLException { String result = null; if(this.conn == null) { String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", userName, password, config); + conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, BaseDbWriter.SYSTEM_DB, config); } ResultSet rs = this.conn.prepareStatement(sql).executeQuery(); if(rs != null) { @@ -137,6 +152,7 @@ public String executeQuery(String sql) throws SQLException { } } + //conn.close(); return result; } @@ -162,7 +178,7 @@ public ResultSet executeQueryWithResultSet(String sql) throws SQLException { * @throws SQLException */ public String getClickHouseVersion() throws SQLException { - return this.executeQuery("SELECT VERSION()"); + return this.executeSystemQuery("SELECT VERSION()"); } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java index e24977722..0da20d8ed 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.apache.maven.artifact.versioning.DefaultArtifactVersion; +import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -49,7 +50,7 @@ public String getEngine() { * @param tableName * @return */ - public MutablePair getTableEngine(ClickHouseConnection conn, String databaseName, String tableName) { + public MutablePair getTableEngine(Connection conn, String databaseName, String tableName) { MutablePair result; result = getTableEngineUsingSystemTables(conn, databaseName, tableName); @@ -67,7 +68,7 @@ public MutablePair getTableEngine(ClickHouseConnection con * @param databaseName * @return */ - public boolean checkIfDatabaseExists(ClickHouseConnection conn, String databaseName) throws SQLException { + public boolean checkIfDatabaseExists(Connection conn, String databaseName) throws SQLException { boolean result = false; try (Statement stmt = conn.createStatement()) { @@ -98,7 +99,7 @@ public boolean checkIfDatabaseExists(ClickHouseConnection conn, String databaseN * @param tableName * @return */ - public MutablePair getTableEngineUsingShowTable(ClickHouseConnection conn, String databaseName, + public MutablePair getTableEngineUsingShowTable(Connection conn, String databaseName, String tableName) { MutablePair result = new MutablePair<>(); @@ -197,7 +198,7 @@ else if(createDML.contains(TABLE_ENGINE.REPLACING_MERGE_TREE.getEngine())) { * @param tableName Table Name. * @return TABLE_ENGINE type */ - public MutablePair getTableEngineUsingSystemTables(final ClickHouseConnection conn, final String database, + public MutablePair getTableEngineUsingSystemTables(final Connection conn, final String database, final String tableName) { MutablePair result = new MutablePair<>(); @@ -277,7 +278,7 @@ public boolean checkIfNewReplacingMergeTree(String currentClickHouseVersion) thr * Function to get the column name and isNullable as key/value pair. */ public Map getColumnsIsNullableForTable(String tableName, - ClickHouseConnection conn, + Connection conn, String database) throws SQLException { Map columnsIsNullable = new HashMap<>(); @@ -301,7 +302,7 @@ public Map getColumnsIsNullableForTable(String tableName, * to get the column name and column data type as key/value pair. */ public Map getColumnsDataTypesForTable(String tableName, - ClickHouseConnection conn, + Connection conn, String database, ClickHouseSinkConnectorConfig config) { @@ -351,15 +352,22 @@ public Map getColumnsDataTypesForTable(String tableName, /** * Function to get the ClickHouse server timezone(Defaults to UTC) */ - public ZoneId getServerTimeZone(ClickHouseConnection conn) { + public ZoneId getServerTimeZone(Connection conn) { ZoneId result = ZoneId.of("UTC"); if(conn != null) { - TimeZone serverTimeZone = conn.getServerTimeZone(); - if(serverTimeZone != null) { - result = serverTimeZone.toZoneId(); - } + try { + // Perform a query to get the server timezone + ResultSet rs = conn.prepareStatement("SELECT timezone()").executeQuery(); + if (rs.next()) { + String serverTimeZone = rs.getString(1); + result = ZoneId.of(serverTimeZone); + } + rs.close(); + } catch (Exception e) { + log.error("Error retrieving server timezone", e); } + } return result; } @@ -368,7 +376,7 @@ public ZoneId getServerTimeZone(ClickHouseConnection conn) { * @return */ public Set getAliasAndMaterializedColumnsForTableAndDatabase(String tableName, String databaseName, - ClickHouseConnection conn) throws SQLException { + Connection conn) throws SQLException { Set aliasColumns = new HashSet<>(); String query = "SELECT name FROM system.columns WHERE (table = '%s') AND (database = '%s') and " + diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriter.java index 98b9dc5c7..8d99b166b 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriter.java @@ -7,10 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.HashMap; import java.util.Map; @@ -30,7 +27,7 @@ public DbKafkaOffsetWriter( String userName, String password, ClickHouseSinkConnectorConfig config, - ClickHouseConnection connection + Connection connection ) { super(hostName, port, database, userName, password, config, connection); diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java index 457532268..3ab550151 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java @@ -5,7 +5,7 @@ import com.altinity.clickhouse.sink.connector.db.operations.ClickHouseAutoCreateTable; import com.altinity.clickhouse.sink.connector.db.operations.ClickHouseCreateDatabase; import com.altinity.clickhouse.sink.connector.model.ClickHouseStruct; -import com.clickhouse.jdbc.ClickHouseConnection; +import io.debezium.storage.jdbc.offset.JdbcOffsetBackingStoreConfig; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.tuple.MutablePair; @@ -13,10 +13,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.sql.SQLException; import java.util.LinkedHashMap; import java.util.Map; +import static io.debezium.storage.jdbc.JdbcCommonConfig.CONFIGURATION_FIELD_PREFIX_STRING; +import static io.debezium.storage.jdbc.offset.JdbcOffsetBackingStoreConfig.OFFSET_STORAGE_PREFIX; + /** * Class that abstracts all functionality * related to interacting with Clickhouse DB. @@ -71,7 +75,7 @@ public DbWriter( String password, ClickHouseSinkConnectorConfig config, ClickHouseStruct record, - ClickHouseConnection connection + Connection connection ) { // Base class initiates connection using JDBC. super(hostName, port, database, userName, password, config, connection); @@ -87,7 +91,12 @@ public DbWriter( } DBMetadata metadata = new DBMetadata(); - createOffsetSchemaHistoryDatabase(); + String offsetStorageDatabaseName = getOffsetStorageDatabaseName(); + if(offsetStorageDatabaseName != null) { + createDestinationDatabase(offsetStorageDatabaseName); + } + // ToDO: create destination database if not exists + createDestinationDatabase(database); MutablePair response = metadata.getTableEngine(this.conn, database, tableName); this.engine = response.getLeft(); @@ -154,36 +163,59 @@ public DbWriter( } } + public String getOffsetStorageDatabaseName() { + + String offsetSchemaHistoryTable = null; + try { + offsetSchemaHistoryTable = config.getString(OFFSET_STORAGE_PREFIX + JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); + } catch(Exception e) { + log.error("***** Error retrieving offset store configuration ****", e); + } + if(offsetSchemaHistoryTable == null || offsetSchemaHistoryTable.isEmpty() == true) { + log.warn("Skipping creating offset schema history table as the query was not provided in configuration"); + return null; + } + String offsetStorageDatabaseNameArray[] = offsetSchemaHistoryTable.split("\\."); + if(offsetStorageDatabaseNameArray.length <= 2) { + log.warn("Skipping creating offset schema history table as the query was not provided in configuration"); + return null; + } + String offsetStorageDatabaseName = offsetStorageDatabaseNameArray[0]; + String offsetStorageTableName = offsetStorageDatabaseNameArray[1]; + + return offsetStorageDatabaseName; + } // Create offset/schema history storage database. - public void createOffsetSchemaHistoryDatabase() { + public void createDestinationDatabase(String databaseName) { + DBMetadata metadata = new DBMetadata(); try { - if (false == metadata.checkIfDatabaseExists(this.conn, database)) { - new ClickHouseCreateDatabase().createNewDatabase(this.conn, database); + if (false == metadata.checkIfDatabaseExists(this.conn, databaseName)) { + new ClickHouseCreateDatabase().createNewDatabase(this.conn, databaseName); } } catch(Exception e) { int maxRetries = 0; final int MAX_RETRIES = 5; - log.error("Error creating Database: " + database); + log.error("Error creating Database: " + databaseName); // Keep retrying to createNewDatabase until Max number of retries is reached. boolean createDatabaseFailed = false; while(maxRetries++ > MAX_RETRIES) { try { Thread.sleep(maxRetries * 5000); - if (false == metadata.checkIfDatabaseExists(this.conn, database)) { - new ClickHouseCreateDatabase().createNewDatabase(this.conn, database); + if (false == metadata.checkIfDatabaseExists(this.conn, databaseName)) { + new ClickHouseCreateDatabase().createNewDatabase(this.conn, databaseName); createDatabaseFailed = true; break; } } catch (Exception ex) { - log.error("Retry Number: " + maxRetries + "of" + MAX_RETRIES + " Error creating Database: " + database); + log.error("Retry Number: " + maxRetries + "of" + MAX_RETRIES + " Error creating Database: " + databaseName); } } // if maxRetries exceeded, throw runtime exception. if(createDatabaseFailed == false) { - throw new RuntimeException("Error creating Database: " + database); + throw new RuntimeException("Error creating Database: " + databaseName); } } } @@ -203,27 +235,6 @@ public boolean wasTableMetaDataRetrieved() { return result; } - - - - /** - * Function to check if the column is of DateTime64 - * from the column type(string name) - * - * @param columnType - * @return true if its DateTime64, false otherwise. - */ - public static boolean isColumnDateTime64(String columnType) { - //ClickHouseDataType dt = ClickHouseDataType.of(columnType); - //ToDo: Figure out a way to get the ClickHouseDataType - // from column name. - boolean result = false; - if (columnType.contains("DateTime64")) { - result = true; - } - return result; - } - public Map getColumnNameToDataTypeMap() { return this.columnNameToDataTypeMap; } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/HikariDbSource.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/HikariDbSource.java new file mode 100644 index 000000000..fb4272356 --- /dev/null +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/HikariDbSource.java @@ -0,0 +1,108 @@ +package com.altinity.clickhouse.sink.connector.db; + +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; +import com.altinity.clickhouse.sink.connector.common.Metrics; +import com.clickhouse.jdbc.ClickHouseDataSource; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.sql.Connection; +import java.util.HashMap; +import java.util.Map; + +// Singleton class(one per database) +public class HikariDbSource { + private static Map instance = new HashMap<>(); + + private static Map connectionPool = new HashMap<>(); + //private static HikariDbSource instance; + + private static final Logger log = LogManager.getLogger(HikariDbSource.class); + //private HikariDataSource dataSource; + private String databaseName; + + // private constructor + private HikariDbSource(ClickHouseDataSource dataSource, String databaseName) { + // this.createConnectionPool(dataSource, databaseName); + } + + + public static HikariDataSource getInstance(ClickHouseDataSource dataSource, String databaseName, + ClickHouseSinkConnectorConfig config) { + + if(instance.containsKey(databaseName)) { + return instance.get(databaseName); + } else { + HikariDataSource hikariDataSource = createConnectionPool(dataSource, databaseName, config); + instance.put(databaseName, hikariDataSource); + } + return instance.get(databaseName); + } + + private static HikariDataSource createConnectionPool(ClickHouseDataSource chDataSource, + String databaseName, ClickHouseSinkConnectorConfig config) { + // pass the clickhouse config to create the datasource + + int maxPoolSize = config.getInt(ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_MAX_SIZE.toString()); + long poolConnectionTimeout = config.getLong(ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_TIMEOUT.toString()); + int minIdle = config.getInt(ClickHouseSinkConnectorConfigVariables.CONNECTION_POOL_MIN_IDLE.toString()); + + HikariConfig poolConfig = new HikariConfig(); + poolConfig.setPoolName("clickhouse" + "-" + databaseName); + String jdbcUrl = String.format("jdbc:ch:{hostname}:{port}/%s?insert_quorum=auto&server_time_zone&server_version=22.13.1.24495", databaseName); + poolConfig.setJdbcUrl(jdbcUrl); + poolConfig.setDriverClassName("com.clickhouse.jdbc.ClickHouseDriver"); // Ensure driver is set + // poolConfig.setUsername(dataSource.getConnection().getCurrentUser()); // Optional, if already in JDBC URL + // poolConfig.setPassword(dataSource.getConnection().()); // Optional, if already in JDBC URL + poolConfig.setConnectionTimeout(poolConnectionTimeout); + poolConfig.setMaximumPoolSize(maxPoolSize); + //poolConfig.setMinimumIdle(minIdle); + poolConfig.setIdleTimeout(2_000L); + poolConfig.setMaxLifetime(300_000L); + poolConfig.setDataSource(chDataSource); + + HikariDataSource dataSource = new HikariDataSource(poolConfig); + + PrometheusMeterRegistry meterRegistry = Metrics.meterRegistry(); + + if(meterRegistry != null) { + dataSource.setMetricRegistry(meterRegistry); + } + return dataSource; + } + + public static void close() { + + if(instance != null) { + for(HikariDataSource hikariDataSource: instance.values()) { + try { + hikariDataSource.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + instance.clear(); + } + } + + public static void closeDatabaseConnection(String databaseName) { + if(instance.containsKey(databaseName)) { + try { + instance.get(databaseName).close(); + } catch (Exception e) { + e.printStackTrace(); + log.error("Error closing database connection pool", e); + } + } + } + + public static void printConnectionInfo() { + for(HikariDataSource hikariDataSource: instance.values()) { + log.debug("Connection Pool Info: " + hikariDataSource.getPoolName() + " Max Size: " + hikariDataSource.getMaximumPoolSize() + " Active Connections: " + hikariDataSource.getHikariPoolMXBean().getActiveConnections()); + } + } +} diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/GroupInsertQueryWithBatchRecords.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/GroupInsertQueryWithBatchRecords.java index 390804463..ef78926fe 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/GroupInsertQueryWithBatchRecords.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/GroupInsertQueryWithBatchRecords.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.util.*; import static com.altinity.clickhouse.sink.connector.db.batch.CdcOperation.getCdcSectionBasedOnOperation; @@ -37,7 +38,7 @@ public boolean groupQueryWithRecords(List records, queryToRecordsMap, Map partitionToOffsetMap, ClickHouseSinkConnectorConfig config, - String tableName, String databaseName, ClickHouseConnection connection, + String tableName, String databaseName, Connection connection, Map columnNameToDataTypeMap) { boolean result = false; diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java index a7b321d08..bb611e760 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; @@ -69,7 +70,7 @@ public PreparedStatementExecutor(String replacingMergeTreeDeleteColumn, public boolean addToPreparedStatementBatch(String topicName, Map>, List> queryToRecordsMap, BlockMetaData bmd, ClickHouseSinkConnectorConfig config, - ClickHouseConnection conn, + Connection conn, String tableName, Map columnToDataTypeMap, DBMetadata.TABLE_ENGINE engine) throws RuntimeException { @@ -106,7 +107,7 @@ public boolean addToPreparedStatementBatch(String topicName, Map>, List> entry, BlockMetaData bmd, ClickHouseSinkConnectorConfig config, - ClickHouseConnection conn, String tableName, Map columnToDataTypeMap, + Connection conn, String tableName, Map columnToDataTypeMap, DBMetadata.TABLE_ENGINE engine) throws RuntimeException { AtomicBoolean result = new AtomicBoolean(false); diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAlterTable.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAlterTable.java index 0e8224173..b3dfb588f 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAlterTable.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAlterTable.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -53,7 +54,7 @@ public String createAlterTableSyntax(String tableName, Map colNa * @para * m modifiedFields */ - public void alterTable(List modifiedFields, String tableName, ClickHouseConnection connection, Map columnNameToDataTypeMap) { + public void alterTable(List modifiedFields, String tableName, Connection connection, Map columnNameToDataTypeMap) { List missingFieldsInCH = new ArrayList(); // Identify the columns that need to be added/removed in ClickHouse. for(Field f: modifiedFields) { diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java index 0d8b906ad..b3b50003d 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Map; @@ -25,7 +26,7 @@ public class ClickHouseAutoCreateTable extends ClickHouseTableOperationsBase{ private static final Logger log = LogManager.getLogger(ClickHouseAutoCreateTable.class.getName()); public void createNewTable(ArrayList primaryKey, String tableName, String databaseName, Field[] fields, - ClickHouseConnection connection, boolean isNewReplacingMergeTree, + Connection connection, boolean isNewReplacingMergeTree, boolean useReplicatedReplacingMergeTree, String rmtDeleteColumn) throws SQLException { Map colNameToDataTypeMap = this.getColumnNameToCHDataTypeMapping(fields); String createTableQuery = this.createTableSyntax(primaryKey, tableName, databaseName, fields, colNameToDataTypeMap, diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabase.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabase.java index 3e277b94d..c89f47bb2 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabase.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabase.java @@ -2,10 +2,11 @@ import com.clickhouse.jdbc.ClickHouseConnection; +import java.sql.Connection; import java.sql.SQLException; public class ClickHouseCreateDatabase extends ClickHouseTableOperationsBase { - public void createNewDatabase(ClickHouseConnection conn, String dbName) throws SQLException { + public void createNewDatabase(Connection conn, String dbName) throws SQLException { String query = String.format("USE system; CREATE DATABASE IF NOT EXISTS %s; USE %s", dbName, dbName); this.runQuery(query, conn); } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java index cefac1c24..faace9e42 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; @@ -100,7 +101,7 @@ public Map getColumnNameToCHDataTypeMapping(Field[] fields) { * @param query * @param conn */ - public void runQuery(String query, ClickHouseConnection conn) throws SQLException { + public void runQuery(String query, Connection conn) throws SQLException { if(conn == null) { log.error("ClickHouse connection not created"); diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchExecutor.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchExecutor.java index a44787805..abac9e4d3 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchExecutor.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchExecutor.java @@ -2,10 +2,34 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; public class ClickHouseBatchExecutor extends ScheduledThreadPoolExecutor { + boolean isPaused = false; public ClickHouseBatchExecutor(int corePoolSize, ThreadFactory threadFactory) { super(corePoolSize, threadFactory); + + + } + + public void pause() { + isPaused = true; + } + + @Override + public void beforeExecute(Thread t, Runnable r) { + while (isPaused) { + try { + TimeUnit.MILLISECONDS.sleep(100); // Polling + } catch (InterruptedException ie) { + t.interrupt(); + } + } + + } + public void resume() { + isPaused = false; } } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java index 7d199b2fc..993ef7280 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java @@ -4,10 +4,7 @@ import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; import com.altinity.clickhouse.sink.connector.common.Metrics; import com.altinity.clickhouse.sink.connector.common.Utils; -import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; -import com.altinity.clickhouse.sink.connector.db.DBMetadata; -import com.altinity.clickhouse.sink.connector.db.DbKafkaOffsetWriter; -import com.altinity.clickhouse.sink.connector.db.DbWriter; +import com.altinity.clickhouse.sink.connector.db.*; import com.altinity.clickhouse.sink.connector.db.batch.GroupInsertQueryWithBatchRecords; import com.altinity.clickhouse.sink.connector.db.batch.PreparedStatementExecutor; import com.altinity.clickhouse.sink.connector.model.BlockMetaData; @@ -19,6 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.sql.SQLException; import java.time.ZoneId; import java.util.ArrayList; @@ -41,11 +39,11 @@ public class ClickHouseBatchRunnable implements Runnable { // Connection that will be used to create // the debezium storage database. - private ClickHouseConnection systemConnection; + private Connection systemConnection; // For insert batch the database connection has to be the same. // Create a map of database name to ClickHouseConnection. - private Map databaseToConnectionMap = new HashMap<>(); + private Map databaseToConnectionMap = new HashMap<>(); /** * Data structures with state */ @@ -80,7 +78,8 @@ public ClickHouseBatchRunnable(LinkedBlockingQueue> recor //this.topicToRecordsMap = new HashMap<>(); this.dbCredentials = parseDBConfiguration(); - this.systemConnection = createConnection(); + + this.systemConnection = createConnection(BaseDbWriter.SYSTEM_DB); try { @@ -91,17 +90,17 @@ public ClickHouseBatchRunnable(LinkedBlockingQueue> recor } } - private ClickHouseConnection createConnection() { + private Connection createConnection(String databaseName) { String jdbcUrl = BaseDbWriter.getConnectionString(this.dbCredentials.getHostName(), this.dbCredentials.getPort(), "system"); - return BaseDbWriter.createConnection(jdbcUrl, "Sink Connector Lightweight", this.dbCredentials.getUserName(), - this.dbCredentials.getPassword(), config); + return BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, this.dbCredentials.getUserName(), + this.dbCredentials.getPassword(), databaseName, config); } // Function to check if we have already stored a ClickHouseConnection // in the databaseToConnectionMap. - private ClickHouseConnection getClickHouseConnection(String databaseName) { + private Connection getClickHouseConnection(String databaseName) { if (this.databaseToConnectionMap.containsKey(databaseName)) { return this.databaseToConnectionMap.get(databaseName); } @@ -109,8 +108,8 @@ private ClickHouseConnection getClickHouseConnection(String databaseName) { String jdbcUrl = BaseDbWriter.getConnectionString(this.dbCredentials.getHostName(), this.dbCredentials.getPort(), databaseName); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Sink Connector Lightweight", - this.dbCredentials.getUserName(), this.dbCredentials.getPassword(), config); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + this.dbCredentials.getUserName(), this.dbCredentials.getPassword(), databaseName, config); this.databaseToConnectionMap.put(databaseName, conn); return conn; @@ -238,7 +237,7 @@ public String getTableFromTopic(String topicName) { } public DbWriter getDbWriterForTable(String topicName, String tableName, String databaseName, - ClickHouseStruct record, ClickHouseConnection connection) { + ClickHouseStruct record, Connection connection) { DbWriter writer = null; if (this.topicToDbWriterMap.containsKey(topicName)) { @@ -296,7 +295,7 @@ private boolean processRecordsByTopic(String topicName, List r if(this.databaseOverrideMap.containsKey(firstRecord.getDatabase())) databaseName = this.databaseOverrideMap.get(firstRecord.getDatabase()); - ClickHouseConnection databaseConn = getClickHouseConnection(databaseName); + Connection databaseConn = getClickHouseConnection(databaseName); DbWriter writer = getDbWriterForTable(topicName, tableName, databaseName, firstRecord, databaseConn); PreparedStatementExecutor preparedStatementExecutor = new diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchWriter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchWriter.java index ce2f163e5..12bc3d362 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchWriter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchWriter.java @@ -19,6 +19,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Connection; import java.sql.SQLException; import java.time.ZoneId; import java.util.ArrayList; @@ -34,11 +35,11 @@ public class ClickHouseBatchWriter { // Connection that will be used to create // the debezium storage database. - private ClickHouseConnection systemConnection; + private Connection systemConnection; // For insert batch the database connection has to be the same. // Create a map of database name to ClickHouseConnection. - private Map databaseToConnectionMap = new HashMap<>(); + private Map databaseToConnectionMap = new HashMap<>(); private static final Logger log = LogManager.getLogger(ClickHouseBatchWriter.class); @@ -67,7 +68,7 @@ public ClickHouseBatchWriter( //this.topicToRecordsMap = new HashMap<>(); this.dbCredentials = parseDBConfiguration(); - this.systemConnection = createConnection(); + this.systemConnection = createConnection(BaseDbWriter.SYSTEM_DB); try { @@ -78,17 +79,17 @@ public ClickHouseBatchWriter( } } - private ClickHouseConnection createConnection() { + private Connection createConnection(String database) { String jdbcUrl = BaseDbWriter.getConnectionString(this.dbCredentials.getHostName(), this.dbCredentials.getPort(), "system"); return BaseDbWriter.createConnection(jdbcUrl, "Sink Connector Lightweight", this.dbCredentials.getUserName(), - this.dbCredentials.getPassword(), config); + this.dbCredentials.getPassword(), database,config); } // Function to check if we have already stored a ClickHouseConnection // in the databaseToConnectionMap. - private ClickHouseConnection getClickHouseConnection(String databaseName) { + private Connection getClickHouseConnection(String databaseName) { if (this.databaseToConnectionMap.containsKey(databaseName)) { return this.databaseToConnectionMap.get(databaseName); } @@ -96,8 +97,8 @@ private ClickHouseConnection getClickHouseConnection(String databaseName) { String jdbcUrl = BaseDbWriter.getConnectionString(this.dbCredentials.getHostName(), this.dbCredentials.getPort(), databaseName); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Sink Connector Lightweight", - this.dbCredentials.getUserName(), this.dbCredentials.getPassword(), config); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, + this.dbCredentials.getUserName(), this.dbCredentials.getPassword(), databaseName, config); this.databaseToConnectionMap.put(databaseName, conn); return conn; @@ -196,7 +197,7 @@ public String getTableFromTopic(String topicName) { } public DbWriter getDbWriterForTable(String topicName, String tableName, String databaseName, - ClickHouseStruct record, ClickHouseConnection connection) { + ClickHouseStruct record, Connection connection) { DbWriter writer = null; if (this.topicToDbWriterMap.containsKey(topicName)) { @@ -254,7 +255,7 @@ private boolean processRecordsByTopic(String topicName, List r if(this.databaseOverrideMap.containsKey(firstRecord.getDatabase())) databaseName = this.databaseOverrideMap.get(firstRecord.getDatabase()); - ClickHouseConnection databaseConn = getClickHouseConnection(databaseName); + Connection databaseConn = getClickHouseConnection(databaseName); DbWriter writer = getDbWriterForTable(topicName, tableName, databaseName, firstRecord, databaseConn); PreparedStatementExecutor preparedStatementExecutor = new diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapperTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapperTest.java index 88c30c43d..05b803aed 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapperTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapperTest.java @@ -14,10 +14,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.time.ZoneId; import java.util.HashMap; @@ -66,7 +63,8 @@ public void convert() throws SQLException { String password = clickHouseContainer.getPassword(); String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = BaseDbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); BaseDbWriter dbWriter = new BaseDbWriter(dbHostName, port, database, userName, password, null, conn); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java index 336ece636..be649928d 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java @@ -1,6 +1,7 @@ package com.altinity.clickhouse.sink.connector.converters; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; import com.altinity.clickhouse.sink.connector.db.DbWriter; import com.altinity.clickhouse.sink.connector.metadata.DataTypeRange; import com.clickhouse.data.ClickHouseDataType; @@ -12,6 +13,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import java.sql.Connection; import java.sql.PreparedStatement; import java.time.*; import java.time.temporal.ChronoUnit; @@ -251,14 +253,15 @@ public void testBatchArrays() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); String jdbcUrl = DbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn1 = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, config); + Connection conn1 = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, config); DbWriter dbWriter = new DbWriter(hostName, port, database, tableName, userName, password, config, null, conn1); String url = dbWriter.getConnectionString(hostName, port, database); String insertQueryTemplate = "insert into test_ch_jdbc_complex_2(col1, col2, col3, col4, col5, col6) values(?, ?, ?, ?, ?, ?)"; try { ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties); - ClickHouseConnection conn = dataSource.getConnection(userName, password); + Connection conn = dataSource.getConnection(userName, password); PreparedStatement ps = conn.prepareStatement(insertQueryTemplate); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriterTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriterTest.java index 4341a27fa..fbf0b2dbd 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriterTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/BaseDbWriterTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.shaded.com.fasterxml.jackson.databind.annotation.JsonAppend; +import java.sql.Connection; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -21,8 +22,8 @@ public void testSplitJdbcProperties() { props.put(ClickHouseSinkConnectorConfigVariables.JDBC_PARAMETERS.toString(), jdbcProperties); ClickHouseSinkConnectorConfig config = new ClickHouseSinkConnectorConfig(props); - ClickHouseConnection conn = BaseDbWriter.createConnection( - "localhost", "client_1","default", "", config); + Connection conn = BaseDbWriter.createConnection( + "localhost", BaseDbWriter.DATABASE_CLIENT_NAME,"default", "",BaseDbWriter.SYSTEM_DB, config); Properties properties = new BaseDbWriter( "localhost", 8123, diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DBMetadataTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DBMetadataTest.java index 2a463596a..67ec28056 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DBMetadataTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DBMetadataTest.java @@ -13,6 +13,7 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.testcontainers.utility.MountableFile; +import java.sql.Connection; import java.sql.SQLException; import java.time.ZoneId; import java.util.HashMap; @@ -66,13 +67,14 @@ public void testCheckIfDatabaseExists() throws SQLException { String dbHostName = clickHouseContainer.getHost(); Integer port = clickHouseContainer.getFirstMappedPort(); - String database = "default"; + String database = "system"; String userName = clickHouseContainer.getUsername(); String password = clickHouseContainer.getPassword(); String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); @@ -141,13 +143,14 @@ public void testIsRMTVersionSupported(String clickhouseVersion, boolean result) public void getTestGetServerTimeZone() { String dbHostName = clickHouseContainer.getHost(); Integer port = clickHouseContainer.getFirstMappedPort(); - String database = "default"; + String database = "system"; String userName = clickHouseContainer.getUsername(); String password = clickHouseContainer.getPassword(); String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB,new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); ZoneId serverTimeZone = new DBMetadata().getServerTimeZone(writer.getConnection()); @@ -166,7 +169,8 @@ public void getAliasAndMaterializedColumnsList() throws SQLException { String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); Set aliasColumns = new DBMetadata().getAliasAndMaterializedColumnsForTableAndDatabase("people", "employees2", conn); Assert.assertTrue(aliasColumns.size() == 2); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriterTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriterTest.java index 4ce0d055e..10ddb2bd0 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriterTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbKafkaOffsetWriterTest.java @@ -13,6 +13,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; import java.util.List; @@ -33,13 +34,14 @@ public void testInsertTopicOffsetMetadata() throws SQLException { String dbHostName = clickHouseContainer.getHost(); Integer port = clickHouseContainer.getFirstMappedPort(); - String database = "default"; + String database = "system"; String userName = clickHouseContainer.getUsername(); String password = clickHouseContainer.getPassword(); String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB,new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java index 757453ff1..0ae256317 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java @@ -22,6 +22,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.sql.Connection; import java.sql.PreparedStatement; import java.time.ZoneId; import java.util.*; @@ -49,8 +50,8 @@ public static void init() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, - config); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, config); writer = new DbWriter(hostName, port, database, tableName, userName, password, config, null, conn); } @@ -87,10 +88,6 @@ public void testGetConnectionUrl() { Assert.assertEquals(connectionUrl, "jdbc:clickhouse://remoteClickHouse:8123/employees"); } - @Test - public void testIsColumnTypeDate64() { - boolean result = DbWriter.isColumnDateTime64("Nullable(DateTime64(3))"); - } @Test @Tag("IntegrationTest") public void testGetColumnsDataTypesForTable() { @@ -103,7 +100,8 @@ public void testGetColumnsDataTypesForTable() { String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); @@ -114,7 +112,8 @@ public void testGetColumnsDataTypesForTable() { String database2 = "employees2"; String jdbcUrl2 = BaseDbWriter.getConnectionString(dbHostName, port, database2); - ClickHouseConnection conn2 = DbWriter.createConnection(jdbcUrl2, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn2 = DbWriter.createConnection(jdbcUrl2, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer2 = new DbWriter(dbHostName, port, database2, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn2); Map columnDataTypesMap2 = writer2.getColumnsDataTypesForTable("employees"); @@ -135,11 +134,11 @@ public void testGetEngineType() { String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, - new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); - MutablePair result = new DBMetadata().getTableEngineUsingShowTable(writer.getConnection(), "default", "employees"); + MutablePair result = new DBMetadata().getTableEngineUsingShowTable(writer.getConnection(), "system", "employees"); Assert.assertTrue(result.getLeft() == DBMetadata.TABLE_ENGINE.REPLACING_MERGE_TREE); Assert.assertTrue(result.getRight().equalsIgnoreCase("_version")); @@ -151,7 +150,7 @@ public void testGetEngineType() { Assert.assertTrue(result_employees.getLeft() == DBMetadata.TABLE_ENGINE.REPLACING_MERGE_TREE); Assert.assertTrue(result_employees.getRight().equalsIgnoreCase("_version_employees")); - MutablePair resultProducts = new DBMetadata().getTableEngineUsingShowTable(writer.getConnection(), "default", "products"); + MutablePair resultProducts = new DBMetadata().getTableEngineUsingShowTable(writer.getConnection(), "system", "products"); Assert.assertTrue(resultProducts.getLeft() == DBMetadata.TABLE_ENGINE.COLLAPSING_MERGE_TREE); Assert.assertTrue(resultProducts.getRight().equalsIgnoreCase("sign")); } @@ -161,14 +160,14 @@ public void testGetEngineType() { public void testGetEngineTypeUsingSystemTables() { String dbHostName = clickHouseContainer.getHost(); Integer port = clickHouseContainer.getFirstMappedPort(); - String database = "default"; + String database = "system"; String userName = clickHouseContainer.getUsername(); String password = clickHouseContainer.getPassword(); String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, - new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); MutablePair< DBMetadata.TABLE_ENGINE, String> result = new DBMetadata().getTableEngineUsingSystemTables(writer.getConnection(), @@ -237,7 +236,8 @@ public void testGroupRecords() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); //String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(connectionUrl, "client_1", userName, password, config); + Connection conn = DbWriter.createConnection(connectionUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, config); DbWriter dbWriter = new DbWriter(dbHostName, port, database, tableName, userName, password, config, null, conn); Map>, List> queryToRecordsMap = new HashMap<>(); @@ -281,7 +281,8 @@ public void testGetClickHouseDataType() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, config); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, config); DbWriter dbWriter = new DbWriter(hostName, port, database, tableName, userName, password, config, null, conn); PreparedStatementExecutor preparedStatementExecutor = new PreparedStatementExecutor(null, false, null, null, database, ZoneId.of("UTC")); @@ -312,7 +313,8 @@ public void testBatchArrays() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn2 = DbWriter.createConnection(jdbcUrl, "client_1", userName, "", config); + Connection conn2 = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, "", + BaseDbWriter.SYSTEM_DB, config); DbWriter dbWriter = new DbWriter(hostName, port, database, tableName, userName, "", config, null, conn2); String url = dbWriter.getConnectionString(hostName, port, database); @@ -336,7 +338,7 @@ public void testBatchArrays() { try { ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties); - ClickHouseConnection conn = dataSource.getConnection(userName, ""); + Connection conn = dataSource.getConnection(userName, ""); PreparedStatement ps = conn.prepareStatement(insertQueryTemplate); @@ -377,7 +379,8 @@ public void testBatchInsert() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn2 = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, config); + Connection conn2 = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, config); DbWriter dbWriter = new DbWriter(hostName, port, database, tableName, userName, password, config, null, conn2); String url = dbWriter.getConnectionString(hostName, port, database); @@ -385,7 +388,7 @@ public void testBatchInsert() { String insertQueryTemplate = "insert into employees values(?,?,?,?,?,?)"; try { ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties); - ClickHouseConnection conn = dataSource.getConnection(userName, password); + Connection conn = dataSource.getConnection(userName, password); PreparedStatement ps = conn.prepareStatement(insertQueryTemplate); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java index e8bbe1e9c..34a130ea9 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java @@ -18,6 +18,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; @@ -29,7 +30,7 @@ public class ClickHouseAutoCreateTableTest { static Map columnToDataTypesMap; - static ClickHouseConnection conn; + static Connection conn; @Container private ClickHouseContainer clickHouseContainer = new ClickHouseContainer("clickhouse/clickhouse-server:latest") @@ -55,7 +56,8 @@ static void initialize() { String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, config); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, config); DbWriter writer = new DbWriter(hostName, port, database, tableName, userName, password, config, null, conn); conn = writer.getConnection(); @@ -163,14 +165,15 @@ public void testCreateTableMultiplePrimaryKeys() { public void testCreateNewTable() { String dbHostName = clickHouseContainer.getHost(); Integer port = clickHouseContainer.getFirstMappedPort(); - String database = "default"; + String database = "system"; String userName = clickHouseContainer.getUsername(); String password = clickHouseContainer.getPassword(); String tableName = "employees"; String jdbcUrl = BaseDbWriter.getConnectionString(dbHostName, port, database); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>())); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + BaseDbWriter.SYSTEM_DB, new ClickHouseSinkConnectorConfig(new HashMap<>())); DbWriter writer = new DbWriter(dbHostName, port, database, tableName, userName, password, new ClickHouseSinkConnectorConfig(new HashMap<>()), null, conn); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabaseTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabaseTest.java index 8673118b4..20b6c84cf 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabaseTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseCreateDatabaseTest.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.clickhouse.ClickHouseContainer; +import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -43,21 +44,22 @@ static void initialize() { ClickHouseSinkConnectorConfig config= new ClickHouseSinkConnectorConfig(new HashMap<>()); String jdbcUrl = BaseDbWriter.getConnectionString(hostName, port, systemDb); - ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, config); - dbWriter = new DbWriter(hostName, port, dbName, null, userName, password, config, null, conn); - maintenanceDbWriter = new DbWriter(hostName, port, systemDb, null, userName, password, config, null, conn); + Connection conn = DbWriter.createConnection(jdbcUrl, BaseDbWriter.DATABASE_CLIENT_NAME, userName, password, + DbWriter.SYSTEM_DB, config); + dbWriter = new DbWriter(hostName, port, dbName, "employees", userName, password, config, null, conn); + maintenanceDbWriter = new DbWriter(hostName, port, systemDb, "employees", userName, password, config, null, conn); } @BeforeEach void dropTestDatabase() throws SQLException { - Statement drop = maintenanceDbWriter.getConnection().createStatement(); - drop.executeQuery(String.format("DROP DATABASE IF EXISTS %s", dbName)); +// Statement drop = maintenanceDbWriter.getConnection().createStatement(); +// drop.executeQuery(String.format("DROP DATABASE IF EXISTS %s", dbName)); } @Test public void testCreateNewDatabase() throws SQLException { ClickHouseCreateDatabase act = new ClickHouseCreateDatabase(); - ClickHouseConnection conn = dbWriter.getConnection(); + Connection conn = dbWriter.getConnection(); try { act.createNewDatabase(conn, dbName); } catch(SQLException se) {