From aa129ef6cc2d9a9cfb8bb9a1b6f6bbf95cd5316e Mon Sep 17 00:00:00 2001 From: silenceland Date: Sun, 15 Jan 2023 18:52:46 +0800 Subject: [PATCH 01/22] [Feature][Connector-V2][OceanBase]OceanBase source and sink connector --- .../dialect/oceanbase/OceanBaseDialect.java | 46 +++++ .../oceanbase/OceanBaseDialectFactory.java | 36 ++++ .../oceanbase/OceanBaseJdbcRowConverter.java | 27 +++ .../oceanbase/OceanBaseTypeMapper.java | 176 ++++++++++++++++ .../seatunnel/jdbc/JdbcOceanBasedbIT.java | 192 ++++++++++++++++++ 5 files changed, 477 insertions(+) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java new file mode 100644 index 00000000000..03fad76ffe8 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; + +import java.util.Optional; + +public class OceanBaseDialect implements JdbcDialect { + @Override + public String dialectName() { + return "OceanBase"; + } + + @Override + public JdbcRowConverter getRowConverter() { + return null; + } + + @Override + public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { + return null; + } + + @Override + public Optional getUpsertStatement(String tableName, String[] fieldNames, String[] uniqueKeyFields) { + return Optional.empty(); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java new file mode 100644 index 00000000000..13983fa4592 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; + +import com.google.auto.service.AutoService; + +@AutoService(JdbcDialectFactory.class) +public class OceanBaseDialectFactory implements JdbcDialectFactory { + @Override + public boolean acceptsURL(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public JdbcDialect create() { + return new OceanBaseDialect(); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java new file mode 100644 index 00000000000..5138363b299 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.AbstractJdbcRowConverter; + +public class OceanBaseJdbcRowConverter extends AbstractJdbcRowConverter { + @Override + public String converterName() { + return "OceanBase"; + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java new file mode 100644 index 00000000000..47df3600328 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; + +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +@Slf4j +public class OceanBaseTypeMapper implements JdbcDialectTypeMapper { + private static final Logger LOG = LoggerFactory.getLogger(JdbcDialect.class); + + private static final int PRECISION_MAX = 38; + private static final int SCALE_MAX = 10; + //ref https://www.oceanbase.com/docs/community-observer-cn-10000000000901502 + // ============================data types===================== + private static final String OCEANBASE_UNKNOWN = "UNKNOWN"; + private static final String OCEANBASE_BIT = "BIT"; + + // -------------------------number---------------------------- + private static final String OCEANBASE_BIGINT = "BIGINT"; + private static final String OCEANBASE_INT = "INT"; + private static final String OCEANBASE_INTEGER = "INTEGER"; + private static final String OCEANBASE_SMALLINT = "SMALLINT"; + private static final String OCEANBASE_BOOL = "BOOL"; + private static final String OCEANBASE_BOOLEAN = "BOOLEAN"; + private static final String OCEANBASE_TINYINT = "TINYINT"; + private static final String OCEANBASE_DECIMAL = "DECIMAL"; + private static final String OCEANBASE_FLOAT = "FLOAT"; + private static final String OCEANBASE_DOUBLE = "DOUBLE"; + private static final String OCEANBASE_NUMBER = "NUMBER"; + + // ------------------------------time------------------------- + private static final String OCEANBASE_DATE = "DATE"; + private static final String OCEANBASE_TIME = "TIME"; + private static final String OCEANBASE_DATETIME = "DATETIME"; + private static final String OCEANBASE_TIMESTAMP = "TIMESTAMP"; + private static final String OCEANBASE_YEAR = "YEAR"; + + // -------------------------string---------------------------- + private static final String OCEANBASE_VARCHAR = "VARCHAR"; + private static final String OCEANBASE_VARBINARY = "VARBINARY"; + private static final String OCEANBASE_CHAR = "CHAR"; + private static final String OCEANBASE_BINARY = "BINARY"; + private static final String OCEANBASE_TINYTEXT = "TINYTEXT"; + private static final String OCEANBASE_TEXT = "TEXT"; + private static final String OCEANBASE_MEDIUMTEXT = "MEDIUMTEXT"; + private static final String OCEANBASE_LONGTEXT = "LONGTEXT"; + private static final String OCEANBASE_ENUM = "ENUM"; + private static final String OCEANBASE_SET = "SET"; + private static final String OCEANBASE_JSON = "JSON"; + + // -------------------------blob------------------------------ + private static final String OCEANBASE_TINYBLOB = "TINYBLOB"; + private static final String OCEANBASE_BLOB = "BLOB"; + private static final String OCEANBASE_MEDIUMBLOB = "MEDIUMBLOB"; + private static final String OCEANBASE_LONGBLOB = "LONGBLOB"; + + @Override + public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) throws SQLException { + String oceanBaseType = metadata.getColumnTypeName(colIndex).toUpperCase(); + int precision = metadata.getPrecision(colIndex); + int scale = metadata.getScale(colIndex); + switch (oceanBaseType) { + case OCEANBASE_BIT: + return BasicType.BOOLEAN_TYPE; + + case OCEANBASE_INT: + case OCEANBASE_INTEGER: + case OCEANBASE_YEAR: + return BasicType.INT_TYPE; + + case OCEANBASE_SMALLINT: + return BasicType.SHORT_TYPE; + case OCEANBASE_BIGINT: + return BasicType.LONG_TYPE; + + case OCEANBASE_BOOL: + case OCEANBASE_BOOLEAN: + case OCEANBASE_TINYINT: + return BasicType.BYTE_TYPE; + + case OCEANBASE_DECIMAL: + if (precision > PRECISION_MAX) { + LOG.warn("{} will probably cause value overflow.", OCEANBASE_DECIMAL); + return new DecimalType(PRECISION_MAX, SCALE_MAX); + } + return new DecimalType(precision, scale); + + case OCEANBASE_FLOAT: + return BasicType.FLOAT_TYPE; + case OCEANBASE_DOUBLE: + return BasicType.DOUBLE_TYPE; + case OCEANBASE_NUMBER: + if (scale == 0 && precision == 0) { + return BasicType.DOUBLE_TYPE; + } + if (scale == 0 && precision > 0) { + return BasicType.LONG_TYPE; + } + if (precision > PRECISION_MAX) { + LOG.warn("{} will probably cause value overflow.", OCEANBASE_NUMBER); + return new DecimalType(PRECISION_MAX, SCALE_MAX); + } else { + return new DecimalType(precision, scale); + } + + case OCEANBASE_DATE: + return LocalTimeType.LOCAL_DATE_TYPE; + case OCEANBASE_TIME: + return LocalTimeType.LOCAL_TIME_TYPE; + case OCEANBASE_DATETIME: + case OCEANBASE_TIMESTAMP: + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + + case OCEANBASE_VARCHAR: + case OCEANBASE_VARBINARY: + case OCEANBASE_CHAR: + case OCEANBASE_BINARY: + case OCEANBASE_TINYTEXT: + case OCEANBASE_TEXT: + case OCEANBASE_MEDIUMTEXT: + case OCEANBASE_ENUM: + case OCEANBASE_SET: + case OCEANBASE_JSON: + return BasicType.STRING_TYPE; + case OCEANBASE_LONGTEXT: + LOG.warn( + "Type '{}' has a maximum precision of 50331648 in OceanBase. " + + "Due to limitations in the seatunnel type system, " + + "the precision will be set to 2147483647.", + OCEANBASE_LONGTEXT); + return BasicType.STRING_TYPE; + + case OCEANBASE_TINYBLOB: + case OCEANBASE_BLOB: + case OCEANBASE_MEDIUMBLOB: + case OCEANBASE_LONGBLOB: + return PrimitiveByteArrayType.INSTANCE; + + case OCEANBASE_UNKNOWN: + default: + final String jdbcColumnName = metadata.getColumnName(colIndex); + throw new UnsupportedOperationException( + String.format( + "Doesn't support OceanBase type '%s' on column '%s' yet.", + oceanBaseType, jdbcColumnName)); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java new file mode 100644 index 00000000000..2f21ad5dbd5 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.jdbc.util.JdbcCompareUtil; + +import org.junit.jupiter.api.Assertions; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +public class JdbcOceanBasedbIT extends AbstractJdbcIT { + private static final String DOCKER_IMAGE = "oceanbase/oceanbase-ce"; + private static final String NETWORK_ALIASES = "e2e_oceanbase"; + private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + private static final int PORT = 2881; + private static final String DATABASE = "test"; + private static final String URL = "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true"; + private static final String USERNAME = "root"; + private static final String PASSWORD = ""; + private static final String SOURCE_TABLE = "e2e_ob_source"; + private static final String SINK_TABLE = "e2e_ob_sink"; + private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; + private static final String CONFIG_FILE = "/jdbc_oceanbase_source_and_sink.conf"; + private static final String COLUMN_STRING = "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27"; + + private static final String DDL_SOURCE = "CREATE TABLE `" + DATABASE + "." + SOURCE_TABLE + "` (\n" + + " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" + + " `col2` integer DEFAULT NULL COMMENT '第二列',\n" + + " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + + " `col4` number DEFAULT NULL COMMENT '第四列',\n" + + " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + + " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" + + " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + + " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" + + " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" + + " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" + + " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + + " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + + " `col13` text DEFAULT NULL COMMENT '第十三列',\n" + + " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" + + " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" + + " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" + + " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" + + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" + + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" + + " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" + + " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" + + " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" + + " PRIMARY KEY (`col1`)\n" + + ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '"+ SOURCE_TABLE + "'"; + + + private static final String DDL_SINK = "CREATE TABLE `" + DATABASE + "." + SINK_TABLE + "` (\n" + + " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" + + " `col2` integer DEFAULT NULL COMMENT '第二列',\n" + + " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + + " `col4` number DEFAULT NULL COMMENT '第四列',\n" + + " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + + " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" + + " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + + " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" + + " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" + + " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" + + " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + + " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + + " `col13` text DEFAULT NULL COMMENT '第十三列',\n" + + " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" + + " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" + + " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" + + " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" + + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" + + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" + + " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" + + " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" + + " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" + + " PRIMARY KEY (`col1`)\n" + + ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '"+ SINK_TABLE + "'"; + + private static final String INIT_DATA_SQL = "insert into " + SOURCE_TABLE + " (\n" + + "`col1` ,\n" + + "`col2` ,\n" + + "`col3` ,\n" + + "`col4` ,\n" + + "`col5` ,\n" + + "`col6` ,\n" + + "`col7` ,\n" + + "`col8` ,\n" + + "`col9` ,\n" + + "`col10`,\n" + + "`col11`,\n" + + "`col12`,\n" + + "`col13`,\n" + + "`col14`,\n" + + "`col15`,\n" + + "`col16`,\n" + + "`col17`,\n" + + "`col18`,\n" + + "`col19`,\n" + + "`col20`,\n" + + "`col21`,\n" + + "`col22`,\n" + + "`col23`,\n" + + "`col24`,\n" + + "`col25`,\n" + + "`col26`,\n" + + "`col27`," + + ")values(\n" + + "\t?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?\n" + + ")"; + + + + @Override + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(URL, PORT, DATABASE); + return JdbcCase.builder().dockerImage(DOCKER_IMAGE).networkAliases(NETWORK_ALIASES).containerEnv(containerEnv).driverClass(DRIVER_CLASS) + .host(HOST).port(PORT).localPort(PORT).jdbcTemplate(URL).jdbcUrl(jdbcUrl).userName(USERNAME).password(PASSWORD).dataBase(DATABASE) + .sourceTable(SOURCE_TABLE).driverJar(DRIVER_JAR) + .ddlSource(DDL_SOURCE).ddlSink(DDL_SINK).initDataSql(INIT_DATA_SQL).configFile(CONFIG_FILE).seaTunnelRow(initTestData()).build(); + } + + @Override + void compareResult() throws SQLException, IOException { + try { + Connection connection = initializeJdbcConnection(URL); + assertHasData(SOURCE_TABLE); + assertHasData(SINK_TABLE); + JdbcCompareUtil.compare(connection, String.format("select * from %s.%s limit 1", DATABASE, SOURCE_TABLE), + String.format("select * from %s.%s limit 1", DATABASE, SINK_TABLE), COLUMN_STRING); + + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { + e.printStackTrace(); + } + + + } + + @Override + void clearSinkTable() { + + } + + @Override + SeaTunnelRow initTestData() { + return new SeaTunnelRow( + new Object[]{101,2,3,4,5,6,7,8,9.1,10.1,"十一","十二","十三","十四","十五","十六","十七","十八","十九","二十","二十一","二十二","19700101","00:00:00","19700101","19700101","1970"} + ); + } + + private void assertHasData(String table) { + try (Statement statement = initializeJdbcConnection(URL).createStatement()) { + String sql = String.format("select * from %s.%s limit 1", DATABASE, table); + ResultSet source = statement.executeQuery(sql); + Assertions.assertTrue(source.next()); + } catch (Exception e) { + throw new RuntimeException("test oceanbase server image error", e); + } + } +} From 51b090148237c1a09796ace32b430791963bb892 Mon Sep 17 00:00:00 2001 From: silenceland Date: Sun, 19 Mar 2023 22:18:30 +0800 Subject: [PATCH 02/22] [Feature][Connector-V2][OceanBase]mvn spotless:apply to fix violations --- .../dialect/oceanbase/OceanBaseDialect.java | 3 +- .../oceanbase/OceanBaseTypeMapper.java | 22 +- .../seatunnel/jdbc/JdbcOceanBasedbIT.java | 286 +++++++++++------- 3 files changed, 187 insertions(+), 124 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java index 03fad76ffe8..42e6a14870e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java @@ -40,7 +40,8 @@ public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { } @Override - public Optional getUpsertStatement(String tableName, String[] fieldNames, String[] uniqueKeyFields) { + public Optional getUpsertStatement( + String database, String tableName, String[] fieldNames, String[] uniqueKeyFields) { return Optional.empty(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java index 47df3600328..3f49bc7a602 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java @@ -25,10 +25,11 @@ import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; -import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; + import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -38,7 +39,7 @@ public class OceanBaseTypeMapper implements JdbcDialectTypeMapper { private static final int PRECISION_MAX = 38; private static final int SCALE_MAX = 10; - //ref https://www.oceanbase.com/docs/community-observer-cn-10000000000901502 + // ref https://www.oceanbase.com/docs/community-observer-cn-10000000000901502 // ============================data types===================== private static final String OCEANBASE_UNKNOWN = "UNKNOWN"; private static final String OCEANBASE_BIT = "BIT"; @@ -83,7 +84,8 @@ public class OceanBaseTypeMapper implements JdbcDialectTypeMapper { private static final String OCEANBASE_LONGBLOB = "LONGBLOB"; @Override - public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) throws SQLException { + public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) + throws SQLException { String oceanBaseType = metadata.getColumnTypeName(colIndex).toUpperCase(); int precision = metadata.getPrecision(colIndex); int scale = metadata.getScale(colIndex); @@ -152,10 +154,10 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th return BasicType.STRING_TYPE; case OCEANBASE_LONGTEXT: LOG.warn( - "Type '{}' has a maximum precision of 50331648 in OceanBase. " - + "Due to limitations in the seatunnel type system, " - + "the precision will be set to 2147483647.", - OCEANBASE_LONGTEXT); + "Type '{}' has a maximum precision of 50331648 in OceanBase. " + + "Due to limitations in the seatunnel type system, " + + "the precision will be set to 2147483647.", + OCEANBASE_LONGTEXT); return BasicType.STRING_TYPE; case OCEANBASE_TINYBLOB: @@ -168,9 +170,9 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th default: final String jdbcColumnName = metadata.getColumnName(colIndex); throw new UnsupportedOperationException( - String.format( - "Doesn't support OceanBase type '%s' on column '%s' yet.", - oceanBaseType, jdbcColumnName)); + String.format( + "Doesn't support OceanBase type '%s' on column '%s' yet.", + oceanBaseType, jdbcColumnName)); } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java index 2f21ad5dbd5..c9eac7034e6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java @@ -36,120 +36,154 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; private static final int PORT = 2881; private static final String DATABASE = "test"; - private static final String URL = "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true"; + private static final String URL = + "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true"; private static final String USERNAME = "root"; private static final String PASSWORD = ""; private static final String SOURCE_TABLE = "e2e_ob_source"; private static final String SINK_TABLE = "e2e_ob_sink"; - private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; + private static final String DRIVER_JAR = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; private static final String CONFIG_FILE = "/jdbc_oceanbase_source_and_sink.conf"; - private static final String COLUMN_STRING = "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27"; - - private static final String DDL_SOURCE = "CREATE TABLE `" + DATABASE + "." + SOURCE_TABLE + "` (\n" + - " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" + - " `col2` integer DEFAULT NULL COMMENT '第二列',\n" + - " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + - " `col4` number DEFAULT NULL COMMENT '第四列',\n" + - " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + - " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" + - " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + - " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" + - " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" + - " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" + - " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + - " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + - " `col13` text DEFAULT NULL COMMENT '第十三列',\n" + - " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" + - " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" + - " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" + - " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" + - " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + - " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + - " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" + - " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + - " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + - " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + - " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" + - " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" + - " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" + - " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" + - " PRIMARY KEY (`col1`)\n" + - ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '"+ SOURCE_TABLE + "'"; - - - private static final String DDL_SINK = "CREATE TABLE `" + DATABASE + "." + SINK_TABLE + "` (\n" + - " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" + - " `col2` integer DEFAULT NULL COMMENT '第二列',\n" + - " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + - " `col4` number DEFAULT NULL COMMENT '第四列',\n" + - " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + - " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" + - " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + - " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" + - " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" + - " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" + - " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + - " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + - " `col13` text DEFAULT NULL COMMENT '第十三列',\n" + - " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" + - " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" + - " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" + - " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" + - " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + - " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + - " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" + - " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + - " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + - " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + - " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" + - " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" + - " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" + - " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" + - " PRIMARY KEY (`col1`)\n" + - ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '"+ SINK_TABLE + "'"; - - private static final String INIT_DATA_SQL = "insert into " + SOURCE_TABLE + " (\n" + - "`col1` ,\n" + - "`col2` ,\n" + - "`col3` ,\n" + - "`col4` ,\n" + - "`col5` ,\n" + - "`col6` ,\n" + - "`col7` ,\n" + - "`col8` ,\n" + - "`col9` ,\n" + - "`col10`,\n" + - "`col11`,\n" + - "`col12`,\n" + - "`col13`,\n" + - "`col14`,\n" + - "`col15`,\n" + - "`col16`,\n" + - "`col17`,\n" + - "`col18`,\n" + - "`col19`,\n" + - "`col20`,\n" + - "`col21`,\n" + - "`col22`,\n" + - "`col23`,\n" + - "`col24`,\n" + - "`col25`,\n" + - "`col26`,\n" + - "`col27`," + - ")values(\n" + - "\t?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?\n" + - ")"; - - + private static final String COLUMN_STRING = + "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27"; + + private static final String DDL_SOURCE = + "CREATE TABLE `" + + DATABASE + + "." + + SOURCE_TABLE + + "` (\n" + + " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" + + " `col2` integer DEFAULT NULL COMMENT '第二列',\n" + + " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + + " `col4` number DEFAULT NULL COMMENT '第四列',\n" + + " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + + " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" + + " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + + " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" + + " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" + + " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" + + " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + + " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + + " `col13` text DEFAULT NULL COMMENT '第十三列',\n" + + " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" + + " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" + + " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" + + " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" + + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" + + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" + + " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" + + " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" + + " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" + + " PRIMARY KEY (`col1`)\n" + + ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '" + + SOURCE_TABLE + + "'"; + + private static final String DDL_SINK = + "CREATE TABLE `" + + DATABASE + + "." + + SINK_TABLE + + "` (\n" + + " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" + + " `col2` integer DEFAULT NULL COMMENT '第二列',\n" + + " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + + " `col4` number DEFAULT NULL COMMENT '第四列',\n" + + " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + + " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" + + " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + + " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" + + " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" + + " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" + + " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + + " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + + " `col13` text DEFAULT NULL COMMENT '第十三列',\n" + + " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" + + " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" + + " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" + + " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" + + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" + + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" + + " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" + + " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" + + " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" + + " PRIMARY KEY (`col1`)\n" + + ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '" + + SINK_TABLE + + "'"; + + private static final String INIT_DATA_SQL = + "insert into " + + SOURCE_TABLE + + " (\n" + + "`col1` ,\n" + + "`col2` ,\n" + + "`col3` ,\n" + + "`col4` ,\n" + + "`col5` ,\n" + + "`col6` ,\n" + + "`col7` ,\n" + + "`col8` ,\n" + + "`col9` ,\n" + + "`col10`,\n" + + "`col11`,\n" + + "`col12`,\n" + + "`col13`,\n" + + "`col14`,\n" + + "`col15`,\n" + + "`col16`,\n" + + "`col17`,\n" + + "`col18`,\n" + + "`col19`,\n" + + "`col20`,\n" + + "`col21`,\n" + + "`col22`,\n" + + "`col23`,\n" + + "`col24`,\n" + + "`col25`,\n" + + "`col26`,\n" + + "`col27`," + + ")values(\n" + + "\t?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?\n" + + ")"; @Override JdbcCase getJdbcCase() { Map containerEnv = new HashMap<>(); String jdbcUrl = String.format(URL, PORT, DATABASE); - return JdbcCase.builder().dockerImage(DOCKER_IMAGE).networkAliases(NETWORK_ALIASES).containerEnv(containerEnv).driverClass(DRIVER_CLASS) - .host(HOST).port(PORT).localPort(PORT).jdbcTemplate(URL).jdbcUrl(jdbcUrl).userName(USERNAME).password(PASSWORD).dataBase(DATABASE) - .sourceTable(SOURCE_TABLE).driverJar(DRIVER_JAR) - .ddlSource(DDL_SOURCE).ddlSink(DDL_SINK).initDataSql(INIT_DATA_SQL).configFile(CONFIG_FILE).seaTunnelRow(initTestData()).build(); + return JdbcCase.builder() + .dockerImage(DOCKER_IMAGE) + .networkAliases(NETWORK_ALIASES) + .containerEnv(containerEnv) + .driverClass(DRIVER_CLASS) + .host(HOST) + .port(PORT) + .localPort(PORT) + .jdbcTemplate(URL) + .jdbcUrl(jdbcUrl) + .userName(USERNAME) + .password(PASSWORD) + .dataBase(DATABASE) + .sourceTable(SOURCE_TABLE) + .driverJar(DRIVER_JAR) + .ddlSource(DDL_SOURCE) + .ddlSink(DDL_SINK) + .initDataSql(INIT_DATA_SQL) + .configFile(CONFIG_FILE) + .seaTunnelRow(initTestData()) + .build(); } @Override @@ -158,26 +192,52 @@ void compareResult() throws SQLException, IOException { Connection connection = initializeJdbcConnection(URL); assertHasData(SOURCE_TABLE); assertHasData(SINK_TABLE); - JdbcCompareUtil.compare(connection, String.format("select * from %s.%s limit 1", DATABASE, SOURCE_TABLE), - String.format("select * from %s.%s limit 1", DATABASE, SINK_TABLE), COLUMN_STRING); + JdbcCompareUtil.compare( + connection, + String.format("select * from %s.%s limit 1", DATABASE, SOURCE_TABLE), + String.format("select * from %s.%s limit 1", DATABASE, SINK_TABLE), + COLUMN_STRING); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { e.printStackTrace(); } - - } @Override - void clearSinkTable() { - - } + void clearSinkTable() {} @Override SeaTunnelRow initTestData() { return new SeaTunnelRow( - new Object[]{101,2,3,4,5,6,7,8,9.1,10.1,"十一","十二","十三","十四","十五","十六","十七","十八","十九","二十","二十一","二十二","19700101","00:00:00","19700101","19700101","1970"} - ); + new Object[] { + 101, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9.1, + 10.1, + "十一", + "十二", + "十三", + "十四", + "十五", + "十六", + "十七", + "十八", + "十九", + "二十", + "二十一", + "二十二", + "19700101", + "00:00:00", + "19700101", + "19700101", + "1970" + }); } private void assertHasData(String table) { From 273a9da5b4ac332de9cc52576ce724646aefa1f3 Mon Sep 17 00:00:00 2001 From: silenceland Date: Sun, 16 Apr 2023 23:31:49 +0800 Subject: [PATCH 03/22] [Feature][Connector-V2][OceanBase]add oceanbase conf --- .../seatunnel/jdbc/JdbcOceanBasedbIT.java | 176 ++++++++++-------- .../jdbc_oceanbase_source_and_sink.conf | 61 ++++++ 2 files changed, 162 insertions(+), 75 deletions(-) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/{ => connector-jdbc-e2e-part-2}/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java (69%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java similarity index 69% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java index c9eac7034e6..d458ce62320 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java @@ -18,33 +18,39 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.connectors.seatunnel.jdbc.util.JdbcCompareUtil; +import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; -import org.junit.jupiter.api.Assertions; +import org.apache.commons.lang3.tuple.Pair; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerLoggerFactory; + +import com.google.common.collect.Lists; + +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; public class JdbcOceanBasedbIT extends AbstractJdbcIT { - private static final String DOCKER_IMAGE = "oceanbase/oceanbase-ce"; - private static final String NETWORK_ALIASES = "e2e_oceanbase"; + + private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce"; + + private static final String OCEANBASE_CONTAINER_HOST = "e2e_oceanbaseDb"; private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; private static final int PORT = 2881; - private static final String DATABASE = "test"; + private static final String DATABASE = "seatunnel"; private static final String URL = "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true"; private static final String USERNAME = "root"; - private static final String PASSWORD = ""; + private static final String PASSWORD = "root"; private static final String SOURCE_TABLE = "e2e_ob_source"; private static final String SINK_TABLE = "e2e_ob_sink"; private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; - private static final String CONFIG_FILE = "/jdbc_oceanbase_source_and_sink.conf"; + private static final List CONFIG_FILE = + Lists.newArrayList("/jdbc_oceanbase_source_and_sink.conf"); private static final String COLUMN_STRING = "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27"; @@ -74,7 +80,7 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" - + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一列',\n" + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" @@ -86,7 +92,7 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { + SOURCE_TABLE + "'"; - private static final String DDL_SINK = + private static final String CREATE_SQL = "CREATE TABLE `" + DATABASE + "." @@ -112,7 +118,7 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" - + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一 列',\n" + + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一列',\n" + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" @@ -162,10 +168,11 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { @Override JdbcCase getJdbcCase() { Map containerEnv = new HashMap<>(); + Pair> testDataSet = initTestData(); String jdbcUrl = String.format(URL, PORT, DATABASE); return JdbcCase.builder() - .dockerImage(DOCKER_IMAGE) - .networkAliases(NETWORK_ALIASES) + .dockerImage(OCEANBASE_IMAGE) + .networkAliases(OCEANBASE_CONTAINER_HOST) .containerEnv(containerEnv) .driverClass(DRIVER_CLASS) .host(HOST) @@ -175,78 +182,97 @@ JdbcCase getJdbcCase() { .jdbcUrl(jdbcUrl) .userName(USERNAME) .password(PASSWORD) - .dataBase(DATABASE) + .database(DATABASE) .sourceTable(SOURCE_TABLE) - .driverJar(DRIVER_JAR) - .ddlSource(DDL_SOURCE) - .ddlSink(DDL_SINK) - .initDataSql(INIT_DATA_SQL) + .sinkTable(SINK_TABLE) + .createSql(CREATE_SQL) .configFile(CONFIG_FILE) - .seaTunnelRow(initTestData()) + .insertSql(INIT_DATA_SQL) + .testData(testDataSet) .build(); } @Override - void compareResult() throws SQLException, IOException { - try { - Connection connection = initializeJdbcConnection(URL); - assertHasData(SOURCE_TABLE); - assertHasData(SINK_TABLE); - JdbcCompareUtil.compare( - connection, - String.format("select * from %s.%s limit 1", DATABASE, SOURCE_TABLE), - String.format("select * from %s.%s limit 1", DATABASE, SINK_TABLE), - COLUMN_STRING); - - } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { - e.printStackTrace(); - } + void compareResult() {} + + @Override + String driverUrl() { + return DRIVER_JAR; } @Override - void clearSinkTable() {} + Pair> initTestData() { + + String[] fieldNames = + new String[] { + "col1", "col2", "col3", "col4", "col5", "col6", "col7", "col8", "col9", "col10", + "col11", "col12", "col13", "col14", "col15", "col16", "col17", "col18", "col19", + "col20", "col21", "col22", "col23", "col24", "col25", "col26", "col27" + }; + + List rows = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + SeaTunnelRow row = + new SeaTunnelRow( + new Object[] { + 100 + i, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9.1, + 10.1, + "十一", + "十二", + "十三", + "十四", + "十五", + "十六", + "十七", + "十八", + "十九", + "二十", + "二十一", + "二十二", + "19700101", + "00:00:00", + "19700101", + "19700101", + "1970" + }); + + rows.add(row); + } + + return Pair.of(fieldNames, rows); + } @Override - SeaTunnelRow initTestData() { - return new SeaTunnelRow( - new Object[] { - 101, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9.1, - 10.1, - "十一", - "十二", - "十三", - "十四", - "十五", - "十六", - "十七", - "十八", - "十九", - "二十", - "二十一", - "二十二", - "19700101", - "00:00:00", - "19700101", - "19700101", - "1970" - }); + GenericContainer initContainer() { + GenericContainer container = + new GenericContainer<>(OCEANBASE_IMAGE) + .withNetwork(NETWORK) + .withNetworkAliases(OCEANBASE_CONTAINER_HOST) + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger(OCEANBASE_IMAGE))); + + container.setPortBindings(Lists.newArrayList(String.format("%s:%s", PORT, PORT))); + + return container; } - private void assertHasData(String table) { - try (Statement statement = initializeJdbcConnection(URL).createStatement()) { - String sql = String.format("select * from %s.%s limit 1", DATABASE, table); - ResultSet source = statement.executeQuery(sql); - Assertions.assertTrue(source.next()); + @Override + protected void createSchemaIfNeeded() { + String sql = "CREATE DATABASE " + DATABASE; + try { + connection.prepareStatement(sql).executeUpdate(); } catch (Exception e) { - throw new RuntimeException("test oceanbase server image error", e); + throw new SeaTunnelRuntimeException( + JdbcITErrorCode.CREATE_TABLE_FAILED, "Fail to execute sql " + sql, e); } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_source_and_sink.conf new file mode 100644 index 00000000000..fa885f5c1cb --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_source_and_sink.conf @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Jdbc { + driver = com.mysql.cj.jdbc.Driver + url = "jdbc:mysql://e2e_oceanbaseDb:2881/seatunnel?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" + user = root + password = root + query = "select col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27 from seatunnel.e2e_ob_source" + } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource +} + +sink { + Assert { + rules = + { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 100 + }, + { + rule_type = MIN_ROW + rule_value = 100 + } + ] + } + } + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert +} From a18844f58ae0f566d5abbc59f0dd9def61a34261 Mon Sep 17 00:00:00 2001 From: silenceland Date: Mon, 17 Apr 2023 08:50:10 +0800 Subject: [PATCH 04/22] [Feature][Connector-V2][OceanBase]fix column float or double --- .../seatunnel/jdbc/JdbcOceanBasedbIT.java | 152 +++++------------- 1 file changed, 37 insertions(+), 115 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java index d458ce62320..82234393ae2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java @@ -42,127 +42,46 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { private static final int PORT = 2881; private static final String DATABASE = "seatunnel"; private static final String URL = - "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true"; + "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true&_enable_convert_real_to_decimal=true"; private static final String USERNAME = "root"; - private static final String PASSWORD = "root"; + private static final String PASSWORD = ""; private static final String SOURCE_TABLE = "e2e_ob_source"; private static final String SINK_TABLE = "e2e_ob_sink"; private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; private static final List CONFIG_FILE = Lists.newArrayList("/jdbc_oceanbase_source_and_sink.conf"); - private static final String COLUMN_STRING = - "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27"; - - private static final String DDL_SOURCE = - "CREATE TABLE `" - + DATABASE - + "." - + SOURCE_TABLE - + "` (\n" - + " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" - + " `col2` integer DEFAULT NULL COMMENT '第二列',\n" - + " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" - + " `col4` number DEFAULT NULL COMMENT '第四列',\n" - + " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" - + " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" - + " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" - + " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" - + " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" - + " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" - + " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" - + " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" - + " `col13` text DEFAULT NULL COMMENT '第十三列',\n" - + " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" - + " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" - + " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" - + " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" - + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" - + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" - + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" - + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一列',\n" - + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" - + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" - + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" - + " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" - + " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" - + " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" - + " PRIMARY KEY (`col1`)\n" - + ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '" - + SOURCE_TABLE - + "'"; private static final String CREATE_SQL = - "CREATE TABLE `" - + DATABASE - + "." - + SINK_TABLE - + "` (\n" - + " `col1` int(11) NOT NULL AUTO_INCREMENT COMMENT '第一列', \n" - + " `col2` integer DEFAULT NULL COMMENT '第二列',\n" - + " `col3` decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" - + " `col4` number DEFAULT NULL COMMENT '第四列',\n" - + " `col5` tinyint(4) DEFAULT NULL COMMENT '第五列',\n" - + " `col6` smallint(6) DEFAULT NULL COMMENT '第六列',\n" - + " `col7` mediumint(9) DEFAULT NULL COMMENT '第七列',\n" - + " `col8` bigint(20) DEFAULT NULL COMMENT '第八列',\n" - + " `col9` float(10,2) DEFAULT NULL COMMENT '第九列',\n" - + " `col10` double(10,2) DEFAULT NULL COMMENT '第十列',\n" - + " `col11` varchar(10) DEFAULT NULL COMMENT '第十一列',\n" - + " `col12` char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" - + " `col13` text DEFAULT NULL COMMENT '第十三列',\n" - + " `col14` tinytext DEFAULT NULL COMMENT '第十四列',\n" - + " `col15` mediumtext DEFAULT NULL COMMENT '第十五列',\n" - + " `col16` longtext DEFAULT NULL COMMENT '第十六列',\n" - + " `col17` blob DEFAULT NULL COMMENT '第十七列',\n" - + " `col18` tinyblob DEFAULT NULL COMMENT '第十八列',\n" - + " `col19` longblob DEFAULT NULL COMMENT '第十九列',\n" - + " `col20` mediumblob DEFAULT NULL COMMENT '第二十列',\n" - + " `col21` binary(16) DEFAULT NULL COMMENT '第二十一列',\n" - + " `col22` varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" - + " `col23` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" - + " `col24` time DEFAULT NULL COMMENT '第二十四列',\n" - + " `col25` date DEFAULT NULL COMMENT '第二十五列',\n" - + " `col26` datetime DEFAULT NULL COMMENT '第二十六列',\n" - + " `col27` year(4) DEFAULT NULL COMMENT '第二十七列',\n" - + " PRIMARY KEY (`col1`)\n" - + ") AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.0' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 COMMENT = '" - + SINK_TABLE - + "'"; - - private static final String INIT_DATA_SQL = - "insert into " - + SOURCE_TABLE + "CREATE TABLE %s\n" + " (\n" - + "`col1` ,\n" - + "`col2` ,\n" - + "`col3` ,\n" - + "`col4` ,\n" - + "`col5` ,\n" - + "`col6` ,\n" - + "`col7` ,\n" - + "`col8` ,\n" - + "`col9` ,\n" - + "`col10`,\n" - + "`col11`,\n" - + "`col12`,\n" - + "`col13`,\n" - + "`col14`,\n" - + "`col15`,\n" - + "`col16`,\n" - + "`col17`,\n" - + "`col18`,\n" - + "`col19`,\n" - + "`col20`,\n" - + "`col21`,\n" - + "`col22`,\n" - + "`col23`,\n" - + "`col24`,\n" - + "`col25`,\n" - + "`col26`,\n" - + "`col27`," - + ")values(\n" - + "\t?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?\n" + + " col1 INT PRIMARY KEY COMMENT '第一列', \n" + + " col2 integer DEFAULT NULL COMMENT '第二列',\n" + + " col3 decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" + + " col4 number DEFAULT NULL COMMENT '第四列',\n" + + " col5 tinyint(4) DEFAULT NULL COMMENT '第五列',\n" + + " col6 smallint(6) DEFAULT NULL COMMENT '第六列',\n" + + " col7 mediumint(9) DEFAULT NULL COMMENT '第七列',\n" + + " col8 bigint(20) DEFAULT NULL COMMENT '第八列',\n" + + " col9 varchar(10) DEFAULT NULL COMMENT '第九列',\n" + + " col10 varchar(10) DEFAULT NULL COMMENT '第十列',\n" + + " col11 varchar(10) DEFAULT NULL COMMENT '第十一列',\n" + + " col12 char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" + + " col13 text DEFAULT NULL COMMENT '第十三列',\n" + + " col14 tinytext DEFAULT NULL COMMENT '第十四列',\n" + + " col15 mediumtext DEFAULT NULL COMMENT '第十五列',\n" + + " col16 longtext DEFAULT NULL COMMENT '第十六列',\n" + + " col17 blob DEFAULT NULL COMMENT '第十七列',\n" + + " col18 tinyblob DEFAULT NULL COMMENT '第十八列',\n" + + " col19 longblob DEFAULT NULL COMMENT '第十九列',\n" + + " col20 mediumblob DEFAULT NULL COMMENT '第二十列',\n" + + " col21 binary(16) DEFAULT NULL COMMENT '第二十一列',\n" + + " col22 varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" + + " col23 timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" + + " col24 time DEFAULT NULL COMMENT '第二十四列',\n" + + " col25 date DEFAULT NULL COMMENT '第二十五列',\n" + + " col26 datetime DEFAULT NULL COMMENT '第二十六列',\n" + + " col27 year(4) DEFAULT NULL COMMENT '第二十七列'\n" + ")"; @Override @@ -170,6 +89,9 @@ JdbcCase getJdbcCase() { Map containerEnv = new HashMap<>(); Pair> testDataSet = initTestData(); String jdbcUrl = String.format(URL, PORT, DATABASE); + String[] fieldNames = testDataSet.getKey(); + String insertSql = insertTable(DATABASE, SOURCE_TABLE, fieldNames); + return JdbcCase.builder() .dockerImage(OCEANBASE_IMAGE) .networkAliases(OCEANBASE_CONTAINER_HOST) @@ -187,7 +109,7 @@ JdbcCase getJdbcCase() { .sinkTable(SINK_TABLE) .createSql(CREATE_SQL) .configFile(CONFIG_FILE) - .insertSql(INIT_DATA_SQL) + .insertSql(insertSql) .testData(testDataSet) .build(); } @@ -223,8 +145,8 @@ Pair> initTestData() { 6, 7, 8, - 9.1, - 10.1, + "9.1", + "10.1", "十一", "十二", "十三", @@ -267,7 +189,7 @@ GenericContainer initContainer() { @Override protected void createSchemaIfNeeded() { - String sql = "CREATE DATABASE " + DATABASE; + String sql = "CREATE DATABASE IF NOT EXISTS " + DATABASE; try { connection.prepareStatement(sql).executeUpdate(); } catch (Exception e) { From 7e881a3f62d70cafea461200add10f50da7d64b6 Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Thu, 20 Apr 2023 10:12:45 +0800 Subject: [PATCH 05/22] [Feature][Jdbc-Connector] Add OceanBase Connector --- seatunnel-connectors-v2/connector-jdbc/pom.xml | 12 +++++++++++- .../seatunnel/jdbc/catalog/AbstractJdbcCatalog.java | 2 +- .../seatunnel/jdbc/config/JdbcConnectionConfig.java | 12 ++++++++++++ .../seatunnel/jdbc/config/JdbcOptions.java | 1 + .../seatunnel/jdbc/config/JdbcSourceConfig.java | 2 ++ .../jdbc/internal/dialect/JdbcDialectFactory.java | 4 +++- .../jdbc/internal/dialect/JdbcDialectLoader.java | 9 +++------ .../jdbc/internal/dialect/db2/DB2DialectFactory.java | 4 +++- .../jdbc/internal/dialect/dm/DmdbDialectFactory.java | 4 +++- .../dialect/gbase8a/Gbase8aDialectFactory.java | 4 +++- .../dialect/greenplum/GreenplumDialectFactory.java | 4 +++- .../internal/dialect/mysql/MySqlDialectFactory.java | 4 +++- .../dialect/oracle/OracleDialectFactory.java | 4 +++- .../dialect/phoenix/PhoenixDialectFactory.java | 4 +++- .../dialect/psql/PostgresDialectFactory.java | 4 +++- .../dialect/redshift/RedshiftDialectFactory.java | 4 +++- .../dialect/saphana/SapHanaDialectFactory.java | 4 +++- .../dialect/sqlite/SqliteDialectFactory.java | 4 +++- .../dialect/sqlserver/SqlServerDialectFactory.java | 4 +++- .../dialect/tablestore/TablestoreDialectFactory.java | 4 +++- .../dialect/teradata/TeradataDialectFactory.java | 4 +++- .../dialect/vertica/VerticaDialectFactory.java | 4 +++- .../connectors/seatunnel/jdbc/sink/JdbcSink.java | 2 +- .../seatunnel/jdbc/sink/JdbcSinkFactory.java | 2 +- .../connectors/seatunnel/jdbc/source/JdbcSource.java | 4 +++- .../seatunnel/jdbc/source/JdbcSourceFactory.java | 2 +- .../example/engine/SeaTunnelEngineExample.java | 2 +- .../example/spark/v2/SeaTunnelApiExample.java | 2 +- 28 files changed, 86 insertions(+), 30 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index f92f605a4eb..29f845c76da 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -44,6 +44,7 @@ 2.1.0.9 2.14.7 12.0.3-0 + 1.1.10 @@ -129,6 +130,12 @@ ${vertica.version} provided + + com.alipay.oceanbase + oceanbase-client + ${oceanbase.version} + provided + @@ -144,7 +151,10 @@ mysql mysql-connector-java - + + com.alipay.oceanbase + oceanbase-client + org.postgresql postgresql diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java index 4fb68775f5e..7a81d65421b 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java @@ -92,7 +92,7 @@ public AbstractJdbcCatalog( this.baseUrl = baseUrl.endsWith("/") ? baseUrl : baseUrl + "/"; this.defaultUrl = urlInfo.getOrigin(); this.suffix = urlInfo.getSuffix(); - this.jdbcDialect = JdbcDialectLoader.load(this.baseUrl); + this.jdbcDialect = JdbcDialectLoader.load(this.baseUrl ,Optional.empty()); } @Override diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java index ab007291171..31039b97480 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java @@ -27,6 +27,7 @@ public class JdbcConnectionConfig implements Serializable { public String url; public String driverName; + public String driverType; public int connectionCheckTimeoutSeconds = JdbcOptions.CONNECTION_CHECK_TIMEOUT_SEC.defaultValue(); public int maxRetries = JdbcOptions.MAX_RETRIES.defaultValue(); @@ -48,6 +49,7 @@ public class JdbcConnectionConfig implements Serializable { public static JdbcConnectionConfig of(ReadonlyConfig config) { JdbcConnectionConfig.Builder builder = JdbcConnectionConfig.builder(); builder.url(config.get(JdbcOptions.URL)); + builder.driverType(config.get(JdbcOptions.DRIVER_TYPE)); builder.driverName(config.get(JdbcOptions.DRIVER)); builder.autoCommit(config.get(JdbcOptions.AUTO_COMMIT)); builder.maxRetries(config.get(JdbcOptions.MAX_RETRIES)); @@ -73,6 +75,10 @@ public String getDriverName() { return driverName; } + public Optional getDriverType() { + return Optional.ofNullable(driverType); + } + public boolean isAutoCommit() { return autoCommit; } @@ -120,6 +126,7 @@ public static JdbcConnectionConfig.Builder builder() { public static final class Builder { private String url; private String driverName; + private String driverType; private int connectionCheckTimeoutSeconds = JdbcOptions.CONNECTION_CHECK_TIMEOUT_SEC.defaultValue(); private int maxRetries = JdbcOptions.MAX_RETRIES.defaultValue(); @@ -145,6 +152,10 @@ public Builder driverName(String driverName) { return this; } + public Builder driverType(String driverType) { + this.driverType = driverType; + return this; + } public Builder connectionCheckTimeoutSeconds(int connectionCheckTimeoutSeconds) { this.connectionCheckTimeoutSeconds = connectionCheckTimeoutSeconds; return this; @@ -205,6 +216,7 @@ public JdbcConnectionConfig build() { jdbcConnectionConfig.batchSize = this.batchSize; jdbcConnectionConfig.batchIntervalMs = this.batchIntervalMs; jdbcConnectionConfig.driverName = this.driverName; + jdbcConnectionConfig.driverType = this.driverType; jdbcConnectionConfig.maxRetries = this.maxRetries; jdbcConnectionConfig.password = this.password; jdbcConnectionConfig.connectionCheckTimeoutSeconds = this.connectionCheckTimeoutSeconds; diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java index d297fbc9e29..e43a4fd5486 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java @@ -35,6 +35,7 @@ public interface JdbcOptions { .intType() .defaultValue(30) .withDescription("connection check time second"); + Option DRIVER_TYPE = Options.key("driver_type").stringType().noDefaultValue().withDescription("driver_type"); Option MAX_RETRIES = Options.key("max_retries").intType().defaultValue(0).withDescription("max_retired"); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java index 7ff4da1f1c8..e20c96994cb 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java @@ -32,6 +32,7 @@ public class JdbcSourceConfig implements Serializable { private JdbcConnectionConfig jdbcConnectionConfig; public String query; + public String driverType; private String partitionColumn; private Long partitionUpperBound; private Long partitionLowerBound; @@ -43,6 +44,7 @@ public static JdbcSourceConfig of(ReadonlyConfig config) { builder.jdbcConnectionConfig(JdbcConnectionConfig.of(config)); builder.query(config.get(JdbcOptions.QUERY)); builder.fetchSize(config.get(JdbcOptions.FETCH_SIZE)); + config.getOptional(JdbcOptions.DRIVER_TYPE).ifPresent(builder::driverType); config.getOptional(JdbcOptions.PARTITION_COLUMN).ifPresent(builder::partitionColumn); config.getOptional(JdbcOptions.PARTITION_UPPER_BOUND) .ifPresent(builder::partitionUpperBound); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java index 5e5ae1b5592..a9364704811 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect; +import java.util.Optional; + /** * A factory to create a specific {@link JdbcDialect} * @@ -33,7 +35,7 @@ public interface JdbcDialectFactory { * @return true if this dialect understands the given URL; false * otherwise. */ - boolean acceptsURL(String url); + boolean acceptsURL(String url, Optional driverTye); /** @return Creates a new instance of the {@link JdbcDialect}. */ JdbcDialect create(); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java index 076a6734b92..04605c39e4a 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java @@ -23,10 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.LinkedList; -import java.util.List; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; +import java.util.*; import java.util.stream.Collectors; /** Utility for working with {@link JdbcDialect}. */ @@ -44,7 +41,7 @@ private JdbcDialectLoader() {} * unambiguously process the given database URL. * @return The loaded dialect. */ - public static JdbcDialect load(String url) { + public static JdbcDialect load(String url, Optional driverTye) { ClassLoader cl = Thread.currentThread().getContextClassLoader(); List foundFactories = discoverFactories(cl); @@ -57,7 +54,7 @@ public static JdbcDialect load(String url) { } final List matchingFactories = - foundFactories.stream().filter(f -> f.acceptsURL(url)).collect(Collectors.toList()); + foundFactories.stream().filter(f -> f.acceptsURL(url, driverTye)).collect(Collectors.toList()); if (matchingFactories.isEmpty()) { throw new JdbcConnectorException( diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java index bbc7536f8c4..465d3bffb5f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java @@ -22,12 +22,14 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link DB2Dialect}. */ @AutoService(JdbcDialectFactory.class) public class DB2DialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:db2:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java index c31d0d2b19e..362a206632d 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java @@ -22,12 +22,14 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link DmdbDialect}. */ @AutoService(JdbcDialectFactory.class) public class DmdbDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:dm:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java index 31787f49740..85b5ba474d3 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java @@ -22,10 +22,12 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class Gbase8aDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:gbase:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java index fb4ca3865f8..d6e70f1774e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java @@ -24,11 +24,13 @@ import com.google.auto.service.AutoService; import lombok.NonNull; +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class GreenplumDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(@NonNull String url) { + public boolean acceptsURL(@NonNull String url, Optional driverTye) { // Support greenplum native driver: com.pivotal.jdbc.GreenplumDriver return url.startsWith("jdbc:pivotal:greenplum:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java index 10047311b93..1bbae8e0dfe 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link MysqlDialect}. */ @AutoService(JdbcDialectFactory.class) public class MySqlDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:mysql:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java index 168dc4d8902..80faa64f570 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link OracleDialect}. */ @AutoService(JdbcDialectFactory.class) public class OracleDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:oracle:thin:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java index 33851af1477..e10faeb2032 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java @@ -23,11 +23,13 @@ import com.google.auto.service.AutoService; import lombok.NonNull; +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class PhoenixDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(@NonNull String url) { + public boolean acceptsURL(@NonNull String url, Optional driverTye) { return url.startsWith("jdbc:phoenix:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java index 963f7385e23..42117706ce1 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java @@ -22,10 +22,12 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class PostgresDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:postgresql:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java index 7470f7a0d41..a269fe37bfd 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java @@ -22,10 +22,12 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class RedshiftDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:redshift:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java index 2d4ea8ad303..237c2eb5c59 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java @@ -23,11 +23,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Dialect Factory of {@link SapHanaDialect} */ @AutoService(JdbcDialectFactory.class) public class SapHanaDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:sap://"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java index 181da6e0a15..b6e0813f2ba 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link SqliteDialect}. */ @AutoService(JdbcDialectFactory.class) public class SqliteDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:sqlite:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java index d8fce3c43c1..23b8dd5de94 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link SqlServerDialect}. */ @AutoService(JdbcDialectFactory.class) public class SqlServerDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:sqlserver:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java index a7285da5a8c..79e9eae73cd 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link TablestoreDialect}. */ @AutoService(JdbcDialectFactory.class) public class TablestoreDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:ots:https:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java index 70a4492868f..e42324cb5fc 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class TeradataDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:teradata:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java index 3a602de940f..11efd403bf2 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java @@ -22,11 +22,13 @@ import com.google.auto.service.AutoService; +import java.util.Optional; + /** Factory for {@link VerticaDialect}. */ @AutoService(JdbcDialectFactory.class) public class VerticaDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url) { + public boolean acceptsURL(String url, Optional driverTye) { return url.startsWith("jdbc:vertica:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java index 2cce860d2eb..1aa4bbe9b2f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java @@ -101,7 +101,7 @@ public String getPluginName() { public void prepare(Config pluginConfig) throws PrepareFailException { this.config = ReadonlyConfig.fromConfig(pluginConfig); this.jdbcSinkConfig = JdbcSinkConfig.of(config); - this.dialect = JdbcDialectLoader.load(jdbcSinkConfig.getJdbcConnectionConfig().getUrl()); + this.dialect = JdbcDialectLoader.load(jdbcSinkConfig.getJdbcConnectionConfig().getUrl(), jdbcSinkConfig.getJdbcConnectionConfig().getDriverType()); this.dataSaveMode = DataSaveMode.KEEP_SCHEMA_AND_DATA; } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java index c8bbe71e44d..4d922cba016 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java @@ -74,7 +74,7 @@ public TableSink createSink(TableFactoryContext context) { } final ReadonlyConfig options = config; JdbcSinkConfig sinkConfig = JdbcSinkConfig.of(config); - JdbcDialect dialect = JdbcDialectLoader.load(sinkConfig.getJdbcConnectionConfig().getUrl()); + JdbcDialect dialect = JdbcDialectLoader.load(sinkConfig.getJdbcConnectionConfig().getUrl(), sinkConfig.getJdbcConnectionConfig().getDriverType()); return () -> new JdbcSink( options, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java index 39deac1efec..88415c51256 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.source; +import lombok.extern.slf4j.Slf4j; import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.api.common.PrepareFailException; @@ -53,6 +54,7 @@ @AutoService(SeaTunnelSource.class) @NoArgsConstructor +@Slf4j public class JdbcSource implements SeaTunnelSource, SupportParallelism, @@ -99,7 +101,7 @@ public void prepare(Config pluginConfig) throws PrepareFailException { new SimpleJdbcConnectionProvider(jdbcSourceConfig.getJdbcConnectionConfig()); this.query = jdbcSourceConfig.getQuery(); this.jdbcDialect = - JdbcDialectLoader.load(jdbcSourceConfig.getJdbcConnectionConfig().getUrl()); + JdbcDialectLoader.load(jdbcSourceConfig.getJdbcConnectionConfig().getUrl(), jdbcSourceConfig.getJdbcConnectionConfig().getDriverType()); try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection()) { this.typeInfo = initTableField(connection); this.partitionParameter = diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java index ee41f2e85b4..2e7f3428475 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java @@ -81,7 +81,7 @@ TableSource createSource(TableFactoryContext context) { JdbcConnectionProvider connectionProvider = new SimpleJdbcConnectionProvider(config.getJdbcConnectionConfig()); final String querySql = config.getQuery(); - JdbcDialect dialect = JdbcDialectLoader.load(config.getJdbcConnectionConfig().getUrl()); + JdbcDialect dialect = JdbcDialectLoader.load(config.getJdbcConnectionConfig().getUrl(), config.getJdbcConnectionConfig().getDriverType()); TableSchema tableSchema = catalogTable.getTableSchema(); SeaTunnelRowType rowType = tableSchema.toPhysicalRowDataType(); Optional partitionParameter = diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java index 2a7c25e0830..2c75e7d663f 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -31,7 +31,7 @@ public class SeaTunnelEngineExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/oceabase.conf"; String configFile = getTestConfigFile(configurePath); ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); clientCommandArgs.setConfigFile(configFile); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java index 870ec64895a..4f22c8439b5 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java @@ -26,7 +26,7 @@ public class SeaTunnelApiExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = args.length > 0 ? args[0] : "/examples/spark.batch.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/oceabase.conf"; ExampleUtils.builder(configurePath); } } From 69427283c1d83e82e8e03b1e1daee66fb4b1427c Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Thu, 20 Apr 2023 20:15:54 +0800 Subject: [PATCH 06/22] [Feature][Jdbc-Connector]Add Oceanbase Connector --- .../OceanBaseMySqlDialectFactory.java | 22 +++++++++++++++++++ .../OceanBaseOracleDialectFactory.java | 22 +++++++++++++++++++ .../engine/SeaTunnelEngineExample.java | 2 +- .../example/spark/v2/SeaTunnelApiExample.java | 2 +- 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java new file mode 100644 index 00000000000..7858eee905e --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java @@ -0,0 +1,22 @@ +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import java.util.Optional; +import com.google.auto.service.AutoService; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql.MysqlDialect; + +@AutoService(JdbcDialectFactory.class) +public class OceanBaseMySqlDialectFactory implements JdbcDialectFactory { + @Override + public boolean acceptsURL(String url, Optional driverTye) { + return url.startsWith("jdbc:oceanbase:") + && driverTye.isPresent() + && driverTye.get().equalsIgnoreCase("mysql"); + } + + @Override + public JdbcDialect create() { + return new MysqlDialect(); + } +} \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java new file mode 100644 index 00000000000..b591d1ef66d --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java @@ -0,0 +1,22 @@ +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import java.util.Optional; +import com.google.auto.service.AutoService; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oracle.OracleDialect; + +@AutoService(JdbcDialectFactory.class) +public class OceanBaseOracleDialectFactory implements JdbcDialectFactory { + @Override + public boolean acceptsURL(String url, Optional driverTye) { + return url.startsWith("jdbc:oceanbase:") + && driverTye.isPresent() + && driverTye.get().equalsIgnoreCase("oracle"); + } + + @Override + public JdbcDialect create() { + return new OracleDialect(); + } +} \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java index 2c75e7d663f..2a7c25e0830 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -31,7 +31,7 @@ public class SeaTunnelEngineExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = args.length > 0 ? args[0] : "/examples/oceabase.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; String configFile = getTestConfigFile(configurePath); ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); clientCommandArgs.setConfigFile(configFile); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java index 4f22c8439b5..09c8332985e 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java @@ -26,7 +26,7 @@ public class SeaTunnelApiExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = args.length > 0 ? args[0] : "/examples/oceabase.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; ExampleUtils.builder(configurePath); } } From 4feacd7e8e7cac5ca9ba9459512be94be9a396a5 Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Fri, 21 Apr 2023 17:13:00 +0800 Subject: [PATCH 07/22] =?UTF-8?q?[Feature][Jdbc-Connector]=201=E3=80=81Mod?= =?UTF-8?q?ify=20ci=20and=20code=20style=20error=202=E3=80=81example=20int?= =?UTF-8?q?o=20docs=203=E3=80=81add=20this=20drive=20into?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/connector-v2/sink/Jdbc.md | 5 ++++ docs/en/connector-v2/source/Jdbc.md | 5 ++++ .../jdbc/catalog/AbstractJdbcCatalog.java | 2 +- .../jdbc/config/JdbcConnectionConfig.java | 1 + .../seatunnel/jdbc/config/JdbcOptions.java | 3 ++- .../internal/dialect/JdbcDialectLoader.java | 5 ++-- .../OceanBaseMySqlDialectFactory.java | 25 ++++++++++++++++--- .../OceanBaseOracleDialectFactory.java | 25 ++++++++++++++++--- .../seatunnel/jdbc/sink/JdbcSink.java | 5 +++- .../seatunnel/jdbc/sink/JdbcSinkFactory.java | 5 +++- .../seatunnel/jdbc/source/JdbcSource.java | 6 +++-- .../jdbc/source/JdbcSourceFactory.java | 5 +++- seatunnel-dist/pom.xml | 8 ++++++ .../src/main/assembly/assembly-bin-ci.xml | 1 + 14 files changed, 86 insertions(+), 15 deletions(-) diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md index d5b17c75786..cba41151301 100644 --- a/docs/en/connector-v2/sink/Jdbc.md +++ b/docs/en/connector-v2/sink/Jdbc.md @@ -33,6 +33,7 @@ support `Xa transactions`. You can set `is_exactly_once=true` to enable it. | user | String | No | - | | password | String | No | - | | query | String | No | - | +| driver_type | String | No | - | | database | String | No | - | | table | String | No | - | | primary_keys | Array | No | - | @@ -68,6 +69,10 @@ The URL of the JDBC connection. Refer to a case: jdbc:postgresql://localhost/tes Use this sql write upstream input datas to database. e.g `INSERT ...` +### driver_type [string] + +Use this field to represent the OceanBase driver. e.g 'mysql' + ### database [string] Use this `database` and `table-name` auto-generate sql and receive upstream input datas write to database. diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md index 183a97d3bc8..ff457baf1c7 100644 --- a/docs/en/connector-v2/source/Jdbc.md +++ b/docs/en/connector-v2/source/Jdbc.md @@ -35,6 +35,7 @@ supports query SQL and can achieve projection effect. | user | String | No | - | | password | String | No | - | | query | String | Yes | - | +| driver_type | String | Yes | - | | connection_check_timeout_sec | Int | No | 30 | | partition_column | String | No | - | | partition_upper_bound | Long | No | - | @@ -63,6 +64,10 @@ The URL of the JDBC connection. Refer to a case: jdbc:postgresql://localhost/tes Query statement +### driver_type [string] + +Use this field to represent the OceanBase driver. e.g 'mysql' + ### connection_check_timeout_sec [int] The time in seconds to wait for the database operation used to validate the connection to complete. diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java index fbc779b1683..f88b3efbb63 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java @@ -81,7 +81,7 @@ public AbstractJdbcCatalog( this.defaultUrl = urlInfo.getOrigin(); this.suffix = urlInfo.getSuffix(); - this.jdbcDialect = JdbcDialectLoader.load(this.baseUrl ,Optional.empty()); + this.jdbcDialect = JdbcDialectLoader.load(this.baseUrl, Optional.empty()); } @Override diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java index 31039b97480..328c3505bc2 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java @@ -156,6 +156,7 @@ public Builder driverType(String driverType) { this.driverType = driverType; return this; } + public Builder connectionCheckTimeoutSeconds(int connectionCheckTimeoutSeconds) { this.connectionCheckTimeoutSeconds = connectionCheckTimeoutSeconds; return this; diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java index e43a4fd5486..7bf073008a6 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java @@ -35,7 +35,8 @@ public interface JdbcOptions { .intType() .defaultValue(30) .withDescription("connection check time second"); - Option DRIVER_TYPE = Options.key("driver_type").stringType().noDefaultValue().withDescription("driver_type"); + Option DRIVER_TYPE = + Options.key("driver_type").stringType().noDefaultValue().withDescription("driver_type"); Option MAX_RETRIES = Options.key("max_retries").intType().defaultValue(0).withDescription("max_retired"); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java index 04605c39e4a..2648226972f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java @@ -23,7 +23,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; import java.util.stream.Collectors; /** Utility for working with {@link JdbcDialect}. */ @@ -54,7 +53,9 @@ public static JdbcDialect load(String url, Optional driverTye) { } final List matchingFactories = - foundFactories.stream().filter(f -> f.acceptsURL(url, driverTye)).collect(Collectors.toList()); + foundFactories.stream() + .filter(f -> f.acceptsURL(url, driverTye)) + .collect(Collectors.toList()); if (matchingFactories.isEmpty()) { throw new JdbcConnectorException( diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java index 7858eee905e..056562d3543 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java @@ -1,11 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; -import java.util.Optional; -import com.google.auto.service.AutoService; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql.MysqlDialect; +import com.google.auto.service.AutoService; + +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class OceanBaseMySqlDialectFactory implements JdbcDialectFactory { @Override @@ -19,4 +38,4 @@ public boolean acceptsURL(String url, Optional driverTye) { public JdbcDialect create() { return new MysqlDialect(); } -} \ No newline at end of file +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java index b591d1ef66d..f152866efe2 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java @@ -1,11 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; -import java.util.Optional; -import com.google.auto.service.AutoService; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oracle.OracleDialect; +import com.google.auto.service.AutoService; + +import java.util.Optional; + @AutoService(JdbcDialectFactory.class) public class OceanBaseOracleDialectFactory implements JdbcDialectFactory { @Override @@ -19,4 +38,4 @@ public boolean acceptsURL(String url, Optional driverTye) { public JdbcDialect create() { return new OracleDialect(); } -} \ No newline at end of file +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java index 1aa4bbe9b2f..197a8de0fb5 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java @@ -101,7 +101,10 @@ public String getPluginName() { public void prepare(Config pluginConfig) throws PrepareFailException { this.config = ReadonlyConfig.fromConfig(pluginConfig); this.jdbcSinkConfig = JdbcSinkConfig.of(config); - this.dialect = JdbcDialectLoader.load(jdbcSinkConfig.getJdbcConnectionConfig().getUrl(), jdbcSinkConfig.getJdbcConnectionConfig().getDriverType()); + this.dialect = + JdbcDialectLoader.load( + jdbcSinkConfig.getJdbcConnectionConfig().getUrl(), + jdbcSinkConfig.getJdbcConnectionConfig().getDriverType()); this.dataSaveMode = DataSaveMode.KEEP_SCHEMA_AND_DATA; } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java index 4d922cba016..3696f240e41 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java @@ -74,7 +74,10 @@ public TableSink createSink(TableFactoryContext context) { } final ReadonlyConfig options = config; JdbcSinkConfig sinkConfig = JdbcSinkConfig.of(config); - JdbcDialect dialect = JdbcDialectLoader.load(sinkConfig.getJdbcConnectionConfig().getUrl(), sinkConfig.getJdbcConnectionConfig().getDriverType()); + JdbcDialect dialect = + JdbcDialectLoader.load( + sinkConfig.getJdbcConnectionConfig().getUrl(), + sinkConfig.getJdbcConnectionConfig().getDriverType()); return () -> new JdbcSink( options, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java index 88415c51256..a1434f3b80b 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.source; -import lombok.extern.slf4j.Slf4j; import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.api.common.PrepareFailException; @@ -47,6 +46,7 @@ import com.google.auto.service.AutoService; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import java.sql.Connection; import java.sql.ResultSetMetaData; @@ -101,7 +101,9 @@ public void prepare(Config pluginConfig) throws PrepareFailException { new SimpleJdbcConnectionProvider(jdbcSourceConfig.getJdbcConnectionConfig()); this.query = jdbcSourceConfig.getQuery(); this.jdbcDialect = - JdbcDialectLoader.load(jdbcSourceConfig.getJdbcConnectionConfig().getUrl(), jdbcSourceConfig.getJdbcConnectionConfig().getDriverType()); + JdbcDialectLoader.load( + jdbcSourceConfig.getJdbcConnectionConfig().getUrl(), + jdbcSourceConfig.getJdbcConnectionConfig().getDriverType()); try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection()) { this.typeInfo = initTableField(connection); this.partitionParameter = diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java index 2e7f3428475..bd38a9f6e3f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java @@ -81,7 +81,10 @@ TableSource createSource(TableFactoryContext context) { JdbcConnectionProvider connectionProvider = new SimpleJdbcConnectionProvider(config.getJdbcConnectionConfig()); final String querySql = config.getQuery(); - JdbcDialect dialect = JdbcDialectLoader.load(config.getJdbcConnectionConfig().getUrl(), config.getJdbcConnectionConfig().getDriverType()); + JdbcDialect dialect = + JdbcDialectLoader.load( + config.getJdbcConnectionConfig().getUrl(), + config.getJdbcConnectionConfig().getDriverType()); TableSchema tableSchema = catalogTable.getTableSchema(); SeaTunnelRowType rowType = tableSchema.toPhysicalRowDataType(); Optional partitionParameter = diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index e9fc30f84d1..925970ba32a 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -96,6 +96,7 @@ 5.13.9 17.20.00.12 2.1.0.9 + 1.1.10 @@ -488,6 +489,12 @@ + + com.alipay.oceanbase + oceanbase-client + ${oceanbase.version} + provided + com.aliyun.phoenix ali-phoenix-shaded-thin-client @@ -571,6 +578,7 @@ 1.11.271 provided + org.apache.seatunnel seatunnel-hadoop3-3.1.4-uber diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml index c6f48949ab5..d0694d75818 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml @@ -193,6 +193,7 @@ com.aliyun.openservices:tablestore-jdbc:jar com.teradata.jdbc:terajdbc4:jar com.amazon.redshift:redshift-jdbc42:jar + com.alipay.oceanbase:oceanbase-client:jar ${artifact.file.name} /plugins/jdbc/lib From c2b59a7411aaca6db72ab44d631d22a68b29cb85 Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Fri, 21 Apr 2023 18:13:25 +0800 Subject: [PATCH 08/22] [Feature][Jdbc-Connector] Add java.util.* --- .../seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java index 2648226972f..dfd7ef90d9d 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.*; import java.util.stream.Collectors; /** Utility for working with {@link JdbcDialect}. */ From f007a2bed3d631941b33e6017436379fba1ab707 Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Sun, 23 Apr 2023 10:01:29 +0800 Subject: [PATCH 09/22] changhuyan [Feature][Jdbc-Connector] Modify ci and code style error --- .../seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java index dfd7ef90d9d..aa0eade6ac0 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java @@ -23,7 +23,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; import java.util.stream.Collectors; /** Utility for working with {@link JdbcDialect}. */ From e9c4289d7ae7e1c7948e6972ca7520bee2ef0de0 Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Mon, 24 Apr 2023 20:20:33 +0800 Subject: [PATCH 10/22] [Feature][Jdbc-Connector] Add e2e testcase --- .../connector-jdbc/pom.xml | 6 +- .../src/main/assembly/assembly-bin-ci.xml | 2 +- .../connector-jdbc-e2e-part-2/pom.xml | 5 + .../seatunnel/jdbc/JdbcOceanbaseIT.java | 139 ++++++++++++++++++ ...ysql_source_and_oceanbase_oracle_sink.conf | 35 +++++ 5 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index 29f845c76da..2e57c32c96e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -44,7 +44,7 @@ 2.1.0.9 2.14.7 12.0.3-0 - 1.1.10 + 2.4.2 @@ -131,7 +131,7 @@ provided - com.alipay.oceanbase + com.oceanbase oceanbase-client ${oceanbase.version} provided @@ -152,7 +152,7 @@ mysql-connector-java - com.alipay.oceanbase + com.oceanbase oceanbase-client diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml index d0694d75818..9bc354daec8 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml @@ -193,7 +193,7 @@ com.aliyun.openservices:tablestore-jdbc:jar com.teradata.jdbc:terajdbc4:jar com.amazon.redshift:redshift-jdbc42:jar - com.alipay.oceanbase:oceanbase-client:jar + com.oceanbase:oceanbase-client:jar ${artifact.file.name} /plugins/jdbc/lib diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml index d90881c817b..134a5e31ad1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml @@ -55,5 +55,10 @@ ngdbc test + + com.oceanbase + oceanbase-client + test + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java new file mode 100644 index 00000000000..b74285d85fb --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import org.apache.commons.lang3.tuple.Pair; + +import org.junit.jupiter.api.Disabled; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Disabled("Disabled because it needs user's personal oceanbase account to run this test!") +public class JdbcOceanbaseIT extends AbstractJdbcIT { + private static final String OCEANBASE_IMAGE = "shihd/oceanbase:1.0"; + private static final String OCEANBASE_CONTAINER_HOST = "spark_e2e_oceanbase"; + private static final String OCEANBASE_DATABASE = "testdb"; + + private static final String OCEANBASE_SOURCE = "people"; + private static final String OCEANBASE_SINK = "test3"; + + private static final String OCEANBASE_USERNAME = "root"; + private static final String OCEANBASE_PASSWORD = "abcd"; + private static final int OCEANBASE_CONTAINER_PORT = 2881; + private static final String OCEANBASE_URL = "jdbc:oceanbase://" + HOST + ":%s/%s"; + + private static final String DRIVER_CLASS = "com.alipay.oceanbase.jdbc.Driver"; + + private static final List CONFIG_FILE = + Lists.newArrayList("/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf"); + + private static final String CREATE_SQL = + "CREATE TABLE `people` (\n" + + " `age` varchar(255) DEFAULT NULL,\n" + + " `name` varchar(255) DEFAULT NULL\n" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; + + @Override + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(OCEANBASE_URL, OCEANBASE_CONTAINER_PORT, OCEANBASE_DATABASE); + Pair> testDataSet = initTestData(); + String[] fieldNames = testDataSet.getKey(); + + String insertSql = insertTable(OCEANBASE_DATABASE, OCEANBASE_SOURCE, fieldNames); + + return JdbcCase.builder() + .dockerImage(OCEANBASE_IMAGE) + .networkAliases(OCEANBASE_CONTAINER_HOST) + .containerEnv(containerEnv) + .driverClass(DRIVER_CLASS) + .host(HOST) + .port(OCEANBASE_CONTAINER_PORT) + .localPort(OCEANBASE_CONTAINER_PORT) + .jdbcTemplate(OCEANBASE_URL) + .jdbcUrl(jdbcUrl) + .userName(OCEANBASE_USERNAME) + .password(OCEANBASE_PASSWORD) + .database(OCEANBASE_DATABASE) + .sourceTable(OCEANBASE_SOURCE) + .sinkTable(OCEANBASE_SINK) + .createSql(CREATE_SQL) + .configFile(CONFIG_FILE) + .insertSql(insertSql) + .testData(testDataSet) + .build(); + } + + @Override + void compareResult() {} + + @Override + String driverUrl() { + return "https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.0/oceanbase-client-2.4.0.jar"; + } + + @Override + Pair> initTestData() { + String[] fieldNames = + new String[] { + "age", "name", + }; + + List rows = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + SeaTunnelRow row = + new SeaTunnelRow( + new Object[] { + i, "f_" + i, + }); + rows.add(row); + } + + return Pair.of(fieldNames, rows); + } + + @Override + GenericContainer initContainer() { + DockerImageName imageName = DockerImageName.parse(OCEANBASE_IMAGE); + + GenericContainer container = + new GenericContainer<>(imageName) + .withNetwork(NETWORK) + .withNetworkAliases(OCEANBASE_CONTAINER_HOST) + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger(OCEANBASE_IMAGE))); + + container.setPortBindings( + Lists.newArrayList( + String.format( + "%s:%s", OCEANBASE_CONTAINER_PORT, OCEANBASE_CONTAINER_PORT))); + return container; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf new file mode 100644 index 00000000000..678e6177a2a --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf @@ -0,0 +1,35 @@ +env { + # You can set engine configuration here + job.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" +} + +source { + jdbc{ + # This is a example source plugin **only for test and demonstrate the feature source plugin** + url = "jdbc:oceanbase://spark_e2e_oceanbase:2881/testdb" + driver = "com.alipay.oceanbase.jdbc.Driver" + user = "root" + password = "abcd" + query = "select age,name from people" + driver_type = "mysql" + } +} + +transform { +} + +sink { + jdbc { + url = "jdbc:oceanbase://spark_e2e_oceanbase:2881/testdb" + driver = "com.alipay.oceanbase.jdbc.Driver" + user = "root" + password = "abcd" + driver_type = "oracle" + query = "insert into test3(age,name) values(?,?)" + } +} From cd2171e591bb521b01013da97382672590b60268 Mon Sep 17 00:00:00 2001 From: changhuyan <877018069@qq.com> Date: Tue, 25 Apr 2023 10:25:42 +0800 Subject: [PATCH 11/22] [Feature][Jdbc-Connector] Add e2etest License --- ..._mysql_source_and_oceanbase_oracle_sink.conf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf index 678e6177a2a..2e3bf907f34 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + env { # You can set engine configuration here job.name = "SeaTunnel" From adcd203dd53e8896699561eed539537fc7b0a9b4 Mon Sep 17 00:00:00 2001 From: silenceland Date: Sun, 18 Jun 2023 23:01:45 +0800 Subject: [PATCH 12/22] [Feature][Connector-V2][OceanBase]spotless apply --- .../connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java index 82234393ae2..a5be10ae7af 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java @@ -42,7 +42,13 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { private static final int PORT = 2881; private static final String DATABASE = "seatunnel"; private static final String URL = - "jdbc:mysql://" + HOST + ":" + PORT + "/" + DATABASE + "?createDatabaseIfNotExist=true&_enable_convert_real_to_decimal=true"; + "jdbc:mysql://" + + HOST + + ":" + + PORT + + "/" + + DATABASE + + "?createDatabaseIfNotExist=true&_enable_convert_real_to_decimal=true"; private static final String USERNAME = "root"; private static final String PASSWORD = ""; private static final String SOURCE_TABLE = "e2e_ob_source"; From b23df0fac5e48085e573d28549f5b6100a263c6e Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 28 Jun 2023 13:20:22 +0800 Subject: [PATCH 13/22] merge dialect factory class and revert unnecessary changes --- .../connector-jdbc/pom.xml | 12 +- .../jdbc/catalog/AbstractJdbcCatalog.java | 2 - .../jdbc/config/JdbcConnectionConfig.java | 4 +- .../internal/dialect/JdbcDialectFactory.java | 14 +- .../internal/dialect/JdbcDialectLoader.java | 10 +- .../dialect/db2/DB2DialectFactory.java | 4 +- .../dialect/dm/DmdbDialectFactory.java | 4 +- .../gbase8a/Gbase8aDialectFactory.java | 4 +- .../greenplum/GreenplumDialectFactory.java | 4 +- .../dialect/mysql/MySqlDialectFactory.java | 4 +- .../dialect/oceanbase/OceanBaseDialect.java | 47 ----- .../oceanbase/OceanBaseDialectFactory.java | 17 +- .../oceanbase/OceanBaseJdbcRowConverter.java | 27 --- .../OceanBaseMySqlDialectFactory.java | 41 ---- .../OceanBaseOracleDialectFactory.java | 41 ---- .../oceanbase/OceanBaseTypeMapper.java | 178 ------------------ .../dialect/oracle/OracleDialectFactory.java | 4 +- .../phoenix/PhoenixDialectFactory.java | 4 +- .../dialect/psql/PostgresDialectFactory.java | 4 +- .../redshift/RedshiftDialectFactory.java | 4 +- .../saphana/SapHanaDialectFactory.java | 4 +- .../dialect/sqlite/SqliteDialectFactory.java | 4 +- .../sqlserver/SqlServerDialectFactory.java | 4 +- .../tablestore/TablestoreDialectFactory.java | 4 +- .../teradata/TeradataDialectFactory.java | 4 +- .../vertica/VerticaDialectFactory.java | 4 +- .../seatunnel/jdbc/source/JdbcSource.java | 2 - seatunnel-dist/pom.xml | 8 +- .../src/main/assembly/assembly-bin-ci.xml | 1 - .../connector-jdbc-e2e-part-2/pom.xml | 5 - .../example/spark/v2/SeaTunnelApiExample.java | 2 +- 31 files changed, 50 insertions(+), 421 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java delete mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java delete mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java delete mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java delete mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index bca5a111394..7b451df1208 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -46,7 +46,6 @@ 3.13.29 12.0.3-0 2.5.1 - 2.4.2 @@ -144,12 +143,6 @@ ${vertica.version} provided - - com.oceanbase - oceanbase-client - ${oceanbase.version} - provided - @@ -165,10 +158,7 @@ mysql mysql-connector-java - - com.oceanbase - oceanbase-client - + org.postgresql postgresql diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java index b667c9c5019..28da8143252 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java @@ -82,8 +82,6 @@ public AbstractJdbcCatalog( this.baseUrl = baseUrl.endsWith("/") ? baseUrl : baseUrl + "/"; this.defaultUrl = urlInfo.getOrigin(); this.suffix = urlInfo.getSuffix(); - - this.jdbcDialect = JdbcDialectLoader.load(this.baseUrl, Optional.empty()); } @Override diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java index 328c3505bc2..db2366ae673 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java @@ -75,8 +75,8 @@ public String getDriverName() { return driverName; } - public Optional getDriverType() { - return Optional.ofNullable(driverType); + public String getDriverType() { + return driverType; } public boolean isAutoCommit() { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java index a9364704811..94b4f9bcd3c 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect; -import java.util.Optional; - /** * A factory to create a specific {@link JdbcDialect} * @@ -35,8 +33,18 @@ public interface JdbcDialectFactory { * @return true if this dialect understands the given URL; false * otherwise. */ - boolean acceptsURL(String url, Optional driverTye); + boolean acceptsURL(String url); /** @return Creates a new instance of the {@link JdbcDialect}. */ JdbcDialect create(); + + /** + * Create a {@link JdbcDialect} instance based on the driver type. + * + * @param driverType The driver type + * @return a new instance of {@link JdbcDialect} + */ + default JdbcDialect create(String driverType) { + return create(); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java index aa0eade6ac0..9f4166ed021 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java @@ -25,7 +25,6 @@ import java.util.LinkedList; import java.util.List; -import java.util.Optional; import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import java.util.stream.Collectors; @@ -41,11 +40,12 @@ private JdbcDialectLoader() {} * Loads the unique JDBC Dialect that can handle the given database url. * * @param url A database URL. + * @param driverType The driver type. * @throws IllegalStateException if the loader cannot find exactly one dialect that can * unambiguously process the given database URL. * @return The loaded dialect. */ - public static JdbcDialect load(String url, Optional driverTye) { + public static JdbcDialect load(String url, String driverType) { ClassLoader cl = Thread.currentThread().getContextClassLoader(); List foundFactories = discoverFactories(cl); @@ -58,9 +58,7 @@ public static JdbcDialect load(String url, Optional driverTye) { } final List matchingFactories = - foundFactories.stream() - .filter(f -> f.acceptsURL(url, driverTye)) - .collect(Collectors.toList()); + foundFactories.stream().filter(f -> f.acceptsURL(url)).collect(Collectors.toList()); if (matchingFactories.isEmpty()) { throw new JdbcConnectorException( @@ -92,7 +90,7 @@ public static JdbcDialect load(String url, Optional driverTye) { .collect(Collectors.joining("\n")))); } - return matchingFactories.get(0).create(); + return matchingFactories.get(0).create(driverType); } private static List discoverFactories(ClassLoader classLoader) { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java index 465d3bffb5f..bbc7536f8c4 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java @@ -22,14 +22,12 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link DB2Dialect}. */ @AutoService(JdbcDialectFactory.class) public class DB2DialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:db2:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java index 362a206632d..c31d0d2b19e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialectFactory.java @@ -22,14 +22,12 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link DmdbDialect}. */ @AutoService(JdbcDialectFactory.class) public class DmdbDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:dm:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java index 85b5ba474d3..31787f49740 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/gbase8a/Gbase8aDialectFactory.java @@ -22,12 +22,10 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(JdbcDialectFactory.class) public class Gbase8aDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:gbase:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java index d6e70f1774e..fb4ca3865f8 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/greenplum/GreenplumDialectFactory.java @@ -24,13 +24,11 @@ import com.google.auto.service.AutoService; import lombok.NonNull; -import java.util.Optional; - @AutoService(JdbcDialectFactory.class) public class GreenplumDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(@NonNull String url, Optional driverTye) { + public boolean acceptsURL(@NonNull String url) { // Support greenplum native driver: com.pivotal.jdbc.GreenplumDriver return url.startsWith("jdbc:pivotal:greenplum:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java index 1bbae8e0dfe..10047311b93 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link MysqlDialect}. */ @AutoService(JdbcDialectFactory.class) public class MySqlDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:mysql:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java deleted file mode 100644 index 42e6a14870e..00000000000 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialect.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; - -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; - -import java.util.Optional; - -public class OceanBaseDialect implements JdbcDialect { - @Override - public String dialectName() { - return "OceanBase"; - } - - @Override - public JdbcRowConverter getRowConverter() { - return null; - } - - @Override - public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { - return null; - } - - @Override - public Optional getUpsertStatement( - String database, String tableName, String[] fieldNames, String[] uniqueKeyFields) { - return Optional.empty(); - } -} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java index 13983fa4592..726871467eb 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java @@ -19,18 +19,31 @@ import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql.MysqlDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oracle.OracleDialect; import com.google.auto.service.AutoService; +import javax.annotation.Nonnull; + @AutoService(JdbcDialectFactory.class) public class OceanBaseDialectFactory implements JdbcDialectFactory { @Override public boolean acceptsURL(String url) { - return url.startsWith("jdbc:mysql:"); + return url.startsWith("jdbc:oceanbase:"); } @Override public JdbcDialect create() { - return new OceanBaseDialect(); + throw new UnsupportedOperationException( + "Can't create JdbcDialect without driver type for OceanBase"); + } + + @Override + public JdbcDialect create(@Nonnull String driverType) { + if ("mysql".equalsIgnoreCase(driverType)) { + return new MysqlDialect(); + } + return new OracleDialect(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java deleted file mode 100644 index 5138363b299..00000000000 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseJdbcRowConverter.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; - -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.AbstractJdbcRowConverter; - -public class OceanBaseJdbcRowConverter extends AbstractJdbcRowConverter { - @Override - public String converterName() { - return "OceanBase"; - } -} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java deleted file mode 100644 index 056562d3543..00000000000 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlDialectFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; - -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql.MysqlDialect; - -import com.google.auto.service.AutoService; - -import java.util.Optional; - -@AutoService(JdbcDialectFactory.class) -public class OceanBaseMySqlDialectFactory implements JdbcDialectFactory { - @Override - public boolean acceptsURL(String url, Optional driverTye) { - return url.startsWith("jdbc:oceanbase:") - && driverTye.isPresent() - && driverTye.get().equalsIgnoreCase("mysql"); - } - - @Override - public JdbcDialect create() { - return new MysqlDialect(); - } -} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java deleted file mode 100644 index f152866efe2..00000000000 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseOracleDialectFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; - -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oracle.OracleDialect; - -import com.google.auto.service.AutoService; - -import java.util.Optional; - -@AutoService(JdbcDialectFactory.class) -public class OceanBaseOracleDialectFactory implements JdbcDialectFactory { - @Override - public boolean acceptsURL(String url, Optional driverTye) { - return url.startsWith("jdbc:oceanbase:") - && driverTye.isPresent() - && driverTye.get().equalsIgnoreCase("oracle"); - } - - @Override - public JdbcDialect create() { - return new OracleDialect(); - } -} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java deleted file mode 100644 index 3f49bc7a602..00000000000 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseTypeMapper.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; - -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.DecimalType; -import org.apache.seatunnel.api.table.type.LocalTimeType; -import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; -import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import lombok.extern.slf4j.Slf4j; - -import java.sql.ResultSetMetaData; -import java.sql.SQLException; - -@Slf4j -public class OceanBaseTypeMapper implements JdbcDialectTypeMapper { - private static final Logger LOG = LoggerFactory.getLogger(JdbcDialect.class); - - private static final int PRECISION_MAX = 38; - private static final int SCALE_MAX = 10; - // ref https://www.oceanbase.com/docs/community-observer-cn-10000000000901502 - // ============================data types===================== - private static final String OCEANBASE_UNKNOWN = "UNKNOWN"; - private static final String OCEANBASE_BIT = "BIT"; - - // -------------------------number---------------------------- - private static final String OCEANBASE_BIGINT = "BIGINT"; - private static final String OCEANBASE_INT = "INT"; - private static final String OCEANBASE_INTEGER = "INTEGER"; - private static final String OCEANBASE_SMALLINT = "SMALLINT"; - private static final String OCEANBASE_BOOL = "BOOL"; - private static final String OCEANBASE_BOOLEAN = "BOOLEAN"; - private static final String OCEANBASE_TINYINT = "TINYINT"; - private static final String OCEANBASE_DECIMAL = "DECIMAL"; - private static final String OCEANBASE_FLOAT = "FLOAT"; - private static final String OCEANBASE_DOUBLE = "DOUBLE"; - private static final String OCEANBASE_NUMBER = "NUMBER"; - - // ------------------------------time------------------------- - private static final String OCEANBASE_DATE = "DATE"; - private static final String OCEANBASE_TIME = "TIME"; - private static final String OCEANBASE_DATETIME = "DATETIME"; - private static final String OCEANBASE_TIMESTAMP = "TIMESTAMP"; - private static final String OCEANBASE_YEAR = "YEAR"; - - // -------------------------string---------------------------- - private static final String OCEANBASE_VARCHAR = "VARCHAR"; - private static final String OCEANBASE_VARBINARY = "VARBINARY"; - private static final String OCEANBASE_CHAR = "CHAR"; - private static final String OCEANBASE_BINARY = "BINARY"; - private static final String OCEANBASE_TINYTEXT = "TINYTEXT"; - private static final String OCEANBASE_TEXT = "TEXT"; - private static final String OCEANBASE_MEDIUMTEXT = "MEDIUMTEXT"; - private static final String OCEANBASE_LONGTEXT = "LONGTEXT"; - private static final String OCEANBASE_ENUM = "ENUM"; - private static final String OCEANBASE_SET = "SET"; - private static final String OCEANBASE_JSON = "JSON"; - - // -------------------------blob------------------------------ - private static final String OCEANBASE_TINYBLOB = "TINYBLOB"; - private static final String OCEANBASE_BLOB = "BLOB"; - private static final String OCEANBASE_MEDIUMBLOB = "MEDIUMBLOB"; - private static final String OCEANBASE_LONGBLOB = "LONGBLOB"; - - @Override - public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) - throws SQLException { - String oceanBaseType = metadata.getColumnTypeName(colIndex).toUpperCase(); - int precision = metadata.getPrecision(colIndex); - int scale = metadata.getScale(colIndex); - switch (oceanBaseType) { - case OCEANBASE_BIT: - return BasicType.BOOLEAN_TYPE; - - case OCEANBASE_INT: - case OCEANBASE_INTEGER: - case OCEANBASE_YEAR: - return BasicType.INT_TYPE; - - case OCEANBASE_SMALLINT: - return BasicType.SHORT_TYPE; - case OCEANBASE_BIGINT: - return BasicType.LONG_TYPE; - - case OCEANBASE_BOOL: - case OCEANBASE_BOOLEAN: - case OCEANBASE_TINYINT: - return BasicType.BYTE_TYPE; - - case OCEANBASE_DECIMAL: - if (precision > PRECISION_MAX) { - LOG.warn("{} will probably cause value overflow.", OCEANBASE_DECIMAL); - return new DecimalType(PRECISION_MAX, SCALE_MAX); - } - return new DecimalType(precision, scale); - - case OCEANBASE_FLOAT: - return BasicType.FLOAT_TYPE; - case OCEANBASE_DOUBLE: - return BasicType.DOUBLE_TYPE; - case OCEANBASE_NUMBER: - if (scale == 0 && precision == 0) { - return BasicType.DOUBLE_TYPE; - } - if (scale == 0 && precision > 0) { - return BasicType.LONG_TYPE; - } - if (precision > PRECISION_MAX) { - LOG.warn("{} will probably cause value overflow.", OCEANBASE_NUMBER); - return new DecimalType(PRECISION_MAX, SCALE_MAX); - } else { - return new DecimalType(precision, scale); - } - - case OCEANBASE_DATE: - return LocalTimeType.LOCAL_DATE_TYPE; - case OCEANBASE_TIME: - return LocalTimeType.LOCAL_TIME_TYPE; - case OCEANBASE_DATETIME: - case OCEANBASE_TIMESTAMP: - return LocalTimeType.LOCAL_DATE_TIME_TYPE; - - case OCEANBASE_VARCHAR: - case OCEANBASE_VARBINARY: - case OCEANBASE_CHAR: - case OCEANBASE_BINARY: - case OCEANBASE_TINYTEXT: - case OCEANBASE_TEXT: - case OCEANBASE_MEDIUMTEXT: - case OCEANBASE_ENUM: - case OCEANBASE_SET: - case OCEANBASE_JSON: - return BasicType.STRING_TYPE; - case OCEANBASE_LONGTEXT: - LOG.warn( - "Type '{}' has a maximum precision of 50331648 in OceanBase. " - + "Due to limitations in the seatunnel type system, " - + "the precision will be set to 2147483647.", - OCEANBASE_LONGTEXT); - return BasicType.STRING_TYPE; - - case OCEANBASE_TINYBLOB: - case OCEANBASE_BLOB: - case OCEANBASE_MEDIUMBLOB: - case OCEANBASE_LONGBLOB: - return PrimitiveByteArrayType.INSTANCE; - - case OCEANBASE_UNKNOWN: - default: - final String jdbcColumnName = metadata.getColumnName(colIndex); - throw new UnsupportedOperationException( - String.format( - "Doesn't support OceanBase type '%s' on column '%s' yet.", - oceanBaseType, jdbcColumnName)); - } - } -} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java index 80faa64f570..168dc4d8902 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link OracleDialect}. */ @AutoService(JdbcDialectFactory.class) public class OracleDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:oracle:thin:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java index e10faeb2032..33851af1477 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/phoenix/PhoenixDialectFactory.java @@ -23,13 +23,11 @@ import com.google.auto.service.AutoService; import lombok.NonNull; -import java.util.Optional; - @AutoService(JdbcDialectFactory.class) public class PhoenixDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(@NonNull String url, Optional driverTye) { + public boolean acceptsURL(@NonNull String url) { return url.startsWith("jdbc:phoenix:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java index 42117706ce1..963f7385e23 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialectFactory.java @@ -22,12 +22,10 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(JdbcDialectFactory.class) public class PostgresDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:postgresql:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java index a269fe37bfd..7470f7a0d41 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftDialectFactory.java @@ -22,12 +22,10 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(JdbcDialectFactory.class) public class RedshiftDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:redshift:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java index 237c2eb5c59..2d4ea8ad303 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/saphana/SapHanaDialectFactory.java @@ -23,13 +23,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Dialect Factory of {@link SapHanaDialect} */ @AutoService(JdbcDialectFactory.class) public class SapHanaDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:sap://"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java index b6e0813f2ba..181da6e0a15 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlite/SqliteDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link SqliteDialect}. */ @AutoService(JdbcDialectFactory.class) public class SqliteDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:sqlite:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java index 23b8dd5de94..d8fce3c43c1 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link SqlServerDialect}. */ @AutoService(JdbcDialectFactory.class) public class SqlServerDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:sqlserver:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java index 79e9eae73cd..a7285da5a8c 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/tablestore/TablestoreDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link TablestoreDialect}. */ @AutoService(JdbcDialectFactory.class) public class TablestoreDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:ots:https:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java index e42324cb5fc..70a4492868f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/teradata/TeradataDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(JdbcDialectFactory.class) public class TeradataDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:teradata:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java index 11efd403bf2..3a602de940f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/vertica/VerticaDialectFactory.java @@ -22,13 +22,11 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - /** Factory for {@link VerticaDialect}. */ @AutoService(JdbcDialectFactory.class) public class VerticaDialectFactory implements JdbcDialectFactory { @Override - public boolean acceptsURL(String url, Optional driverTye) { + public boolean acceptsURL(String url) { return url.startsWith("jdbc:vertica:"); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java index a1434f3b80b..a02cbc88605 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java @@ -46,7 +46,6 @@ import com.google.auto.service.AutoService; import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; import java.sql.Connection; import java.sql.ResultSetMetaData; @@ -54,7 +53,6 @@ @AutoService(SeaTunnelSource.class) @NoArgsConstructor -@Slf4j public class JdbcSource implements SeaTunnelSource, SupportParallelism, diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 29ea8b573a7..51d4f4c48a9 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -98,13 +98,13 @@ 17.20.00.12 2.1.0.9 3.13.29 - 1.1.10 3.0.0 2.4.7 3.1.4 1.11.271 + 4.1.89.Final @@ -510,12 +510,6 @@ - - com.alipay.oceanbase - oceanbase-client - ${oceanbase.version} - provided - com.aliyun.phoenix ali-phoenix-shaded-thin-client diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml index f5e8439b136..3a1b5a4177e 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml @@ -202,7 +202,6 @@ com.teradata.jdbc:terajdbc4:jar com.amazon.redshift:redshift-jdbc42:jar net.snowflake.snowflake-jdbc:jar - com.oceanbase:oceanbase-client:jar ${artifact.file.name} /lib diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml index 134a5e31ad1..d90881c817b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml @@ -55,10 +55,5 @@ ngdbc test - - com.oceanbase - oceanbase-client - test - diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java index 09c8332985e..870ec64895a 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java @@ -26,7 +26,7 @@ public class SeaTunnelApiExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/spark.batch.conf"; ExampleUtils.builder(configurePath); } } From 3ee6054191b0d294da5852bff1912d132be57024 Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 28 Jun 2023 15:40:40 +0800 Subject: [PATCH 14/22] rename IT files --- .../{JdbcOceanBasedbIT.java => JdbcOceanBaseMysqlIT.java} | 4 ++-- .../jdbc/{JdbcOceanbaseIT.java => JdbcOceanBaseOracleIT.java} | 4 ++-- ...nd_sink.conf => jdbc_oceanbase_mysql_source_and_sink.conf} | 0 ...e_sink.conf => jdbc_oceanbase_oracle_source_and_sink.conf} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/{JdbcOceanBasedbIT.java => JdbcOceanBaseMysqlIT.java} (98%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/{JdbcOceanbaseIT.java => JdbcOceanBaseOracleIT.java} (97%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/{jdbc_oceanbase_source_and_sink.conf => jdbc_oceanbase_mysql_source_and_sink.conf} (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/{jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf => jdbc_oceanbase_oracle_source_and_sink.conf} (100%) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java similarity index 98% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java index a5be10ae7af..3daea2120c1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBasedbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.Map; -public class JdbcOceanBasedbIT extends AbstractJdbcIT { +public class JdbcOceanBaseMysqlIT extends AbstractJdbcIT { private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce"; @@ -56,7 +56,7 @@ public class JdbcOceanBasedbIT extends AbstractJdbcIT { private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; private static final List CONFIG_FILE = - Lists.newArrayList("/jdbc_oceanbase_source_and_sink.conf"); + Lists.newArrayList("/jdbc_oceanbase_mysql_source_and_sink.conf"); private static final String CREATE_SQL = "CREATE TABLE %s\n" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java similarity index 97% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java index b74285d85fb..89218bbcd2e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanbaseIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java @@ -35,7 +35,7 @@ import java.util.Map; @Disabled("Disabled because it needs user's personal oceanbase account to run this test!") -public class JdbcOceanbaseIT extends AbstractJdbcIT { +public class JdbcOceanBaseOracleIT extends AbstractJdbcIT { private static final String OCEANBASE_IMAGE = "shihd/oceanbase:1.0"; private static final String OCEANBASE_CONTAINER_HOST = "spark_e2e_oceanbase"; private static final String OCEANBASE_DATABASE = "testdb"; @@ -51,7 +51,7 @@ public class JdbcOceanbaseIT extends AbstractJdbcIT { private static final String DRIVER_CLASS = "com.alipay.oceanbase.jdbc.Driver"; private static final List CONFIG_FILE = - Lists.newArrayList("/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf"); + Lists.newArrayList("/jdbc_oceanbase_oracle_source_and_sink.conf"); private static final String CREATE_SQL = "CREATE TABLE `people` (\n" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_source_and_sink.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_oceanbase_oracle_sink.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf From 3d8dd7f5d745a1260bf085a6497c88fc79a2e4c6 Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 28 Jun 2023 15:58:06 +0800 Subject: [PATCH 15/22] update IT cases based on existing mysql/oracle cases --- .../oceanbase/OceanBaseDialectFactory.java | 6 +- .../seatunnel/jdbc/JdbcOceanBaseITBase.java | 101 ++++++ .../seatunnel/jdbc/JdbcOceanBaseMysqlIT.java | 320 ++++++++++-------- .../seatunnel/jdbc/JdbcOceanBaseOracleIT.java | 170 +++++----- .../jdbc_oceanbase_mysql_source_and_sink.conf | 9 +- ...jdbc_oceanbase_oracle_source_and_sink.conf | 44 ++- 6 files changed, 411 insertions(+), 239 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java index 726871467eb..836ece8601e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java @@ -41,9 +41,9 @@ public JdbcDialect create() { @Override public JdbcDialect create(@Nonnull String driverType) { - if ("mysql".equalsIgnoreCase(driverType)) { - return new MysqlDialect(); + if ("oracle".equalsIgnoreCase(driverType)) { + return new OracleDialect(); } - return new OracleDialect(); + return new MysqlDialect(); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java new file mode 100644 index 00000000000..ceac9908b4b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; + +import org.apache.commons.lang3.tuple.Pair; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public abstract class JdbcOceanBaseITBase extends AbstractJdbcIT { + + private static final String OCEANBASE_DATABASE = "seatunnel"; + private static final String OCEANBASE_SOURCE = "source"; + private static final String OCEANBASE_SINK = "sink"; + + private static final String OCEANBASE_JDBC_TEMPLATE = "jdbc:oceanbase://" + HOST + ":%s"; + private static final String OCEANBASE_DRIVER_CLASS = "com.oceanbase.jdbc.Driver"; + + abstract String imageName(); + + abstract String host(); + + abstract int port(); + + abstract String username(); + + abstract String password(); + + abstract List configFile(); + + abstract String createSqlTemplate(); + + @Override + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(OCEANBASE_JDBC_TEMPLATE, port()); + Pair> testDataSet = initTestData(); + String[] fieldNames = testDataSet.getKey(); + + String insertSql = insertTable(OCEANBASE_DATABASE, OCEANBASE_SOURCE, fieldNames); + + return JdbcCase.builder() + .dockerImage(imageName()) + .networkAliases(host()) + .containerEnv(containerEnv) + .driverClass(OCEANBASE_DRIVER_CLASS) + .host(HOST) + .port(port()) + .localPort(port()) + .jdbcTemplate(OCEANBASE_JDBC_TEMPLATE) + .jdbcUrl(jdbcUrl) + .userName(username()) + .password(password()) + .database(OCEANBASE_DATABASE) + .sourceTable(OCEANBASE_SOURCE) + .sinkTable(OCEANBASE_SINK) + .createSql(createSqlTemplate()) + .configFile(configFile()) + .insertSql(insertSql) + .testData(testDataSet) + .build(); + } + + @Override + void compareResult() {} + + @Override + String driverUrl() { + return "https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.3/oceanbase-client-2.4.3.jar"; + } + + @Override + protected void createSchemaIfNeeded() { + String sql = "CREATE DATABASE IF NOT EXISTS " + OCEANBASE_DATABASE; + try { + connection.prepareStatement(sql).executeUpdate(); + } catch (Exception e) { + throw new SeaTunnelRuntimeException( + JdbcITErrorCode.CREATE_TABLE_FAILED, "Fail to execute sql " + sql, e); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java index 3daea2120c1..8297f1d3db0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java @@ -18,160 +18,214 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; import org.apache.commons.lang3.tuple.Pair; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerLoggerFactory; import com.google.common.collect.Lists; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; - -public class JdbcOceanBaseMysqlIT extends AbstractJdbcIT { - - private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce"; - - private static final String OCEANBASE_CONTAINER_HOST = "e2e_oceanbaseDb"; - private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - private static final int PORT = 2881; - private static final String DATABASE = "seatunnel"; - private static final String URL = - "jdbc:mysql://" - + HOST - + ":" - + PORT - + "/" - + DATABASE - + "?createDatabaseIfNotExist=true&_enable_convert_real_to_decimal=true"; - private static final String USERNAME = "root"; - private static final String PASSWORD = ""; - private static final String SOURCE_TABLE = "e2e_ob_source"; - private static final String SINK_TABLE = "e2e_ob_sink"; - private static final String DRIVER_JAR = - "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; - private static final List CONFIG_FILE = - Lists.newArrayList("/jdbc_oceanbase_mysql_source_and_sink.conf"); - - private static final String CREATE_SQL = - "CREATE TABLE %s\n" - + " (\n" - + " col1 INT PRIMARY KEY COMMENT '第一列', \n" - + " col2 integer DEFAULT NULL COMMENT '第二列',\n" - + " col3 decimal(10,2) DEFAULT NULL COMMENT '第三列',\n" - + " col4 number DEFAULT NULL COMMENT '第四列',\n" - + " col5 tinyint(4) DEFAULT NULL COMMENT '第五列',\n" - + " col6 smallint(6) DEFAULT NULL COMMENT '第六列',\n" - + " col7 mediumint(9) DEFAULT NULL COMMENT '第七列',\n" - + " col8 bigint(20) DEFAULT NULL COMMENT '第八列',\n" - + " col9 varchar(10) DEFAULT NULL COMMENT '第九列',\n" - + " col10 varchar(10) DEFAULT NULL COMMENT '第十列',\n" - + " col11 varchar(10) DEFAULT NULL COMMENT '第十一列',\n" - + " col12 char(10) NOT NULL DEFAULT '中文字段' COMMENT '第十二列',\n" - + " col13 text DEFAULT NULL COMMENT '第十三列',\n" - + " col14 tinytext DEFAULT NULL COMMENT '第十四列',\n" - + " col15 mediumtext DEFAULT NULL COMMENT '第十五列',\n" - + " col16 longtext DEFAULT NULL COMMENT '第十六列',\n" - + " col17 blob DEFAULT NULL COMMENT '第十七列',\n" - + " col18 tinyblob DEFAULT NULL COMMENT '第十八列',\n" - + " col19 longblob DEFAULT NULL COMMENT '第十九列',\n" - + " col20 mediumblob DEFAULT NULL COMMENT '第二十列',\n" - + " col21 binary(16) DEFAULT NULL COMMENT '第二十一列',\n" - + " col22 varbinary(16) DEFAULT NULL COMMENT '第二十二列',\n" - + " col23 timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '第二十三列',\n" - + " col24 time DEFAULT NULL COMMENT '第二十四列',\n" - + " col25 date DEFAULT NULL COMMENT '第二十五列',\n" - + " col26 datetime DEFAULT NULL COMMENT '第二十六列',\n" - + " col27 year(4) DEFAULT NULL COMMENT '第二十七列'\n" - + ")"; + +public class JdbcOceanBaseMysqlIT extends JdbcOceanBaseITBase { @Override - JdbcCase getJdbcCase() { - Map containerEnv = new HashMap<>(); - Pair> testDataSet = initTestData(); - String jdbcUrl = String.format(URL, PORT, DATABASE); - String[] fieldNames = testDataSet.getKey(); - String insertSql = insertTable(DATABASE, SOURCE_TABLE, fieldNames); - - return JdbcCase.builder() - .dockerImage(OCEANBASE_IMAGE) - .networkAliases(OCEANBASE_CONTAINER_HOST) - .containerEnv(containerEnv) - .driverClass(DRIVER_CLASS) - .host(HOST) - .port(PORT) - .localPort(PORT) - .jdbcTemplate(URL) - .jdbcUrl(jdbcUrl) - .userName(USERNAME) - .password(PASSWORD) - .database(DATABASE) - .sourceTable(SOURCE_TABLE) - .sinkTable(SINK_TABLE) - .createSql(CREATE_SQL) - .configFile(CONFIG_FILE) - .insertSql(insertSql) - .testData(testDataSet) - .build(); + String imageName() { + return "oceanbase/oceanbase-ce:4.0.0.0"; } @Override - void compareResult() {} + String host() { + return "e2e_oceanbase_mysql"; + } @Override - String driverUrl() { - return DRIVER_JAR; + int port() { + return 2881; } @Override - Pair> initTestData() { + String username() { + return "root"; + } + + @Override + String password() { + return ""; + } + + @Override + List configFile() { + return Lists.newArrayList("/jdbc_oceanbase_mysql_source_and_sink.conf"); + } + @Override + String createSqlTemplate() { + return "CREATE TABLE IF NOT EXISTS %s\n" + + "(\n" + + " `c_bit_1` bit(1) DEFAULT NULL,\n" + + " `c_bit_8` bit(8) DEFAULT NULL,\n" + + " `c_bit_16` bit(16) DEFAULT NULL,\n" + + " `c_bit_32` bit(32) DEFAULT NULL,\n" + + " `c_bit_64` bit(64) DEFAULT NULL,\n" + + " `c_boolean` tinyint(1) DEFAULT NULL,\n" + + " `c_tinyint` tinyint(4) DEFAULT NULL,\n" + + " `c_tinyint_unsigned` tinyint(3) unsigned DEFAULT NULL,\n" + + " `c_smallint` smallint(6) DEFAULT NULL,\n" + + " `c_smallint_unsigned` smallint(5) unsigned DEFAULT NULL,\n" + + " `c_mediumint` mediumint(9) DEFAULT NULL,\n" + + " `c_mediumint_unsigned` mediumint(8) unsigned DEFAULT NULL,\n" + + " `c_int` int(11) DEFAULT NULL,\n" + + " `c_integer` int(11) DEFAULT NULL,\n" + + " `c_bigint` bigint(20) DEFAULT NULL,\n" + + " `c_bigint_unsigned` bigint(20) unsigned DEFAULT NULL,\n" + + " `c_decimal` decimal(20, 0) DEFAULT NULL,\n" + + " `c_decimal_unsigned` decimal(38, 18) DEFAULT NULL,\n" + + " `c_float` float DEFAULT NULL,\n" + + " `c_float_unsigned` float unsigned DEFAULT NULL,\n" + + " `c_double` double DEFAULT NULL,\n" + + " `c_double_unsigned` double unsigned DEFAULT NULL,\n" + + " `c_char` char(1) DEFAULT NULL,\n" + + " `c_tinytext` tinytext,\n" + + " `c_mediumtext` mediumtext,\n" + + " `c_text` text,\n" + + " `c_varchar` varchar(255) DEFAULT NULL,\n" + + " `c_json` json DEFAULT NULL,\n" + + " `c_longtext` longtext,\n" + + " `c_date` date DEFAULT NULL,\n" + + " `c_datetime` datetime DEFAULT NULL,\n" + + " `c_timestamp` timestamp NULL DEFAULT NULL,\n" + + " `c_tinyblob` tinyblob,\n" + + " `c_mediumblob` mediumblob,\n" + + " `c_blob` blob,\n" + + " `c_longblob` longblob,\n" + + " `c_varbinary` varbinary(255) DEFAULT NULL,\n" + + " `c_binary` binary(1) DEFAULT NULL,\n" + + " `c_year` year(4) DEFAULT NULL,\n" + + " `c_int_unsigned` int(10) unsigned DEFAULT NULL,\n" + + " `c_integer_unsigned` int(10) unsigned DEFAULT NULL,\n" + + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" + + " `c_decimal_unsigned_30` DECIMAL(30) unsigned DEFAULT NULL,\n" + + " `c_decimal_30` DECIMAL(30) DEFAULT NULL\n" + + ");"; + }; + + @Override + Pair> initTestData() { String[] fieldNames = new String[] { - "col1", "col2", "col3", "col4", "col5", "col6", "col7", "col8", "col9", "col10", - "col11", "col12", "col13", "col14", "col15", "col16", "col17", "col18", "col19", - "col20", "col21", "col22", "col23", "col24", "col25", "col26", "col27" + "c_bit_1", + "c_bit_8", + "c_bit_16", + "c_bit_32", + "c_bit_64", + "c_boolean", + "c_tinyint", + "c_tinyint_unsigned", + "c_smallint", + "c_smallint_unsigned", + "c_mediumint", + "c_mediumint_unsigned", + "c_int", + "c_integer", + "c_year", + "c_int_unsigned", + "c_integer_unsigned", + "c_bigint", + "c_bigint_unsigned", + "c_decimal", + "c_decimal_unsigned", + "c_float", + "c_float_unsigned", + "c_double", + "c_double_unsigned", + "c_char", + "c_tinytext", + "c_mediumtext", + "c_text", + "c_varchar", + "c_json", + "c_longtext", + "c_date", + "c_datetime", + "c_timestamp", + "c_tinyblob", + "c_mediumblob", + "c_blob", + "c_longblob", + "c_varbinary", + "c_binary", + "c_bigint_30", + "c_decimal_unsigned_30", + "c_decimal_30", }; List rows = new ArrayList<>(); + BigDecimal bigintValue = new BigDecimal("2844674407371055000"); + BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); for (int i = 0; i < 100; i++) { + byte byteArr = Integer.valueOf(i).byteValue(); SeaTunnelRow row = new SeaTunnelRow( new Object[] { - 100 + i, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - "9.1", - "10.1", - "十一", - "十二", - "十三", - "十四", - "十五", - "十六", - "十七", - "十八", - "十九", - "二十", - "二十一", - "二十二", - "19700101", - "00:00:00", - "19700101", - "19700101", - "1970" + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr + }, + i % 2 == 0 ? Boolean.TRUE : Boolean.FALSE, + i, + i, + i, + i, + i, + i, + i, + i, + i, + Long.parseLong("1"), + Long.parseLong("1"), + Long.parseLong("1"), + BigDecimal.valueOf(i, 0), + BigDecimal.valueOf(i, 18), + BigDecimal.valueOf(i, 18), + Float.parseFloat("1.1"), + Float.parseFloat("1.1"), + Double.parseDouble("1.1"), + Double.parseDouble("1.1"), + "f", + String.format("f1_%s", i), + String.format("f1_%s", i), + String.format("f1_%s", i), + String.format("f1_%s", i), + String.format("{\"aa\":\"bb_%s\"}", i), + String.format("f1_%s", i), + Date.valueOf(LocalDate.now()), + Timestamp.valueOf(LocalDateTime.now()), + new Timestamp(System.currentTimeMillis()), + "test".getBytes(), + "test".getBytes(), + "test".getBytes(), + "test".getBytes(), + "test".getBytes(), + "f".getBytes(), + bigintValue.add(BigDecimal.valueOf(i)), + decimalValue.add(BigDecimal.valueOf(i)), + decimalValue.add(BigDecimal.valueOf(i)), }); - rows.add(row); } @@ -181,26 +235,16 @@ Pair> initTestData() { @Override GenericContainer initContainer() { GenericContainer container = - new GenericContainer<>(OCEANBASE_IMAGE) + new GenericContainer<>(imageName()) .withNetwork(NETWORK) - .withNetworkAliases(OCEANBASE_CONTAINER_HOST) + .withNetworkAliases(host()) + .waitingFor(Wait.forLogMessage(".*boot success!.*", 1)) + .withStartupTimeout(Duration.ofMinutes(5)) .withLogConsumer( - new Slf4jLogConsumer( - DockerLoggerFactory.getLogger(OCEANBASE_IMAGE))); + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(imageName()))); - container.setPortBindings(Lists.newArrayList(String.format("%s:%s", PORT, PORT))); + container.setPortBindings(Lists.newArrayList(String.format("%s:%s", port(), port()))); return container; } - - @Override - protected void createSchemaIfNeeded() { - String sql = "CREATE DATABASE IF NOT EXISTS " + DATABASE; - try { - connection.prepareStatement(sql).executeUpdate(); - } catch (Exception e) { - throw new SeaTunnelRuntimeException( - JdbcITErrorCode.CREATE_TABLE_FAILED, "Fail to execute sql " + sql, e); - } - } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java index 89218bbcd2e..56a065e13d1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java @@ -23,86 +23,112 @@ import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.DockerLoggerFactory; import com.google.common.collect.Lists; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.concurrent.TimeUnit; -@Disabled("Disabled because it needs user's personal oceanbase account to run this test!") -public class JdbcOceanBaseOracleIT extends AbstractJdbcIT { - private static final String OCEANBASE_IMAGE = "shihd/oceanbase:1.0"; - private static final String OCEANBASE_CONTAINER_HOST = "spark_e2e_oceanbase"; - private static final String OCEANBASE_DATABASE = "testdb"; +import static org.awaitility.Awaitility.given; - private static final String OCEANBASE_SOURCE = "people"; - private static final String OCEANBASE_SINK = "test3"; +@Disabled("Oracle mode of OceanBase Enterprise Edition does not provide docker environment") +public class JdbcOceanBaseOracleIT extends JdbcOceanBaseITBase { - private static final String OCEANBASE_USERNAME = "root"; - private static final String OCEANBASE_PASSWORD = "abcd"; - private static final int OCEANBASE_CONTAINER_PORT = 2881; - private static final String OCEANBASE_URL = "jdbc:oceanbase://" + HOST + ":%s/%s"; + @Override + String imageName() { + return null; + } - private static final String DRIVER_CLASS = "com.alipay.oceanbase.jdbc.Driver"; + @Override + String host() { + return "e2e_oceanbase_oracle"; + } - private static final List CONFIG_FILE = - Lists.newArrayList("/jdbc_oceanbase_oracle_source_and_sink.conf"); + @Override + int port() { + return 2883; + } - private static final String CREATE_SQL = - "CREATE TABLE `people` (\n" - + " `age` varchar(255) DEFAULT NULL,\n" - + " `name` varchar(255) DEFAULT NULL\n" - + ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; + @Override + String username() { + return "root"; + } @Override - JdbcCase getJdbcCase() { - Map containerEnv = new HashMap<>(); - String jdbcUrl = String.format(OCEANBASE_URL, OCEANBASE_CONTAINER_PORT, OCEANBASE_DATABASE); - Pair> testDataSet = initTestData(); - String[] fieldNames = testDataSet.getKey(); - - String insertSql = insertTable(OCEANBASE_DATABASE, OCEANBASE_SOURCE, fieldNames); - - return JdbcCase.builder() - .dockerImage(OCEANBASE_IMAGE) - .networkAliases(OCEANBASE_CONTAINER_HOST) - .containerEnv(containerEnv) - .driverClass(DRIVER_CLASS) - .host(HOST) - .port(OCEANBASE_CONTAINER_PORT) - .localPort(OCEANBASE_CONTAINER_PORT) - .jdbcTemplate(OCEANBASE_URL) - .jdbcUrl(jdbcUrl) - .userName(OCEANBASE_USERNAME) - .password(OCEANBASE_PASSWORD) - .database(OCEANBASE_DATABASE) - .sourceTable(OCEANBASE_SOURCE) - .sinkTable(OCEANBASE_SINK) - .createSql(CREATE_SQL) - .configFile(CONFIG_FILE) - .insertSql(insertSql) - .testData(testDataSet) - .build(); + String password() { + return ""; } @Override - void compareResult() {} + List configFile() { + return Lists.newArrayList("/jdbc_oceanbase_oracle_source_and_sink.conf"); + } @Override - String driverUrl() { - return "https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.0/oceanbase-client-2.4.0.jar"; + GenericContainer initContainer() { + throw new UnsupportedOperationException(); + } + + @Override + public void startUp() { + jdbcCase = getJdbcCase(); + + given().ignoreExceptions() + .await() + .atMost(360, TimeUnit.SECONDS) + .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); + + createSchemaIfNeeded(); + createNeededTables(); + insertTestData(); + } + + @Override + public String quoteIdentifier(String field) { + return "\"" + field + "\""; + } + + @Override + String createSqlTemplate() { + return "create table %s\n" + + "(\n" + + " VARCHAR_10_COL varchar2(10),\n" + + " CHAR_10_COL char(10),\n" + + " CLOB_COL clob,\n" + + " NUMBER_3_SF_2_DP number(3, 2),\n" + + " INTEGER_COL integer,\n" + + " FLOAT_COL float(10),\n" + + " REAL_COL real,\n" + + " BINARY_FLOAT_COL binary_float,\n" + + " BINARY_DOUBLE_COL binary_double,\n" + + " DATE_COL date,\n" + + " TIMESTAMP_WITH_3_FRAC_SEC_COL timestamp(3),\n" + + " TIMESTAMP_WITH_LOCAL_TZ timestamp with local time zone\n" + + ")"; } @Override Pair> initTestData() { String[] fieldNames = new String[] { - "age", "name", + "VARCHAR_10_COL", + "CHAR_10_COL", + "CLOB_COL", + "NUMBER_3_SF_2_DP", + "INTEGER_COL", + "FLOAT_COL", + "REAL_COL", + "BINARY_FLOAT_COL", + "BINARY_DOUBLE_COL", + "DATE_COL", + "TIMESTAMP_WITH_3_FRAC_SEC_COL", + "TIMESTAMP_WITH_LOCAL_TZ" }; List rows = new ArrayList<>(); @@ -110,30 +136,22 @@ Pair> initTestData() { SeaTunnelRow row = new SeaTunnelRow( new Object[] { - i, "f_" + i, + String.format("f%s", i), + String.format("f%s", i), + String.format("f%s", i), + BigDecimal.valueOf(1.1), + i, + Float.parseFloat("2.2"), + Float.parseFloat("2.2"), + Float.parseFloat("22.2"), + Double.parseDouble("2.2"), + Date.valueOf(LocalDate.now()), + Timestamp.valueOf(LocalDateTime.now()), + Timestamp.valueOf(LocalDateTime.now()) }); rows.add(row); } return Pair.of(fieldNames, rows); } - - @Override - GenericContainer initContainer() { - DockerImageName imageName = DockerImageName.parse(OCEANBASE_IMAGE); - - GenericContainer container = - new GenericContainer<>(imageName) - .withNetwork(NETWORK) - .withNetworkAliases(OCEANBASE_CONTAINER_HOST) - .withLogConsumer( - new Slf4jLogConsumer( - DockerLoggerFactory.getLogger(OCEANBASE_IMAGE))); - - container.setPortBindings( - Lists.newArrayList( - String.format( - "%s:%s", OCEANBASE_CONTAINER_PORT, OCEANBASE_CONTAINER_PORT))); - return container; - } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf index fa885f5c1cb..992d02556e7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf @@ -29,11 +29,12 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** Jdbc { - driver = com.mysql.cj.jdbc.Driver - url = "jdbc:mysql://e2e_oceanbaseDb:2881/seatunnel?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" + driver = com.oceanbase.jdbc.Driver + url = "jdbc:oceanbase://e2e_oceanbase_mysql:2881/seatunnel?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" user = root - password = root - query = "select col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27 from seatunnel.e2e_ob_source" + password = "" + query = "select * from source" + driver_type = "mysql" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf index 2e3bf907f34..3d85c1e5f98 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf @@ -14,26 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### env { - # You can set engine configuration here - job.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } source { jdbc{ # This is a example source plugin **only for test and demonstrate the feature source plugin** - url = "jdbc:oceanbase://spark_e2e_oceanbase:2881/testdb" - driver = "com.alipay.oceanbase.jdbc.Driver" + url = "jdbc:oceanbase://e2e_oceanbase_oracle:2883/seatunnel" + driver = com.oceanbase.jdbc.Driver user = "root" - password = "abcd" - query = "select age,name from people" - driver_type = "mysql" + password = "" + query = "SELECT * FROM source" + driver_type = "oracle" } } @@ -41,12 +42,19 @@ transform { } sink { - jdbc { - url = "jdbc:oceanbase://spark_e2e_oceanbase:2881/testdb" - driver = "com.alipay.oceanbase.jdbc.Driver" - user = "root" - password = "abcd" - driver_type = "oracle" - query = "insert into test3(age,name) values(?,?)" + Assert { + rules = + { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 100 + }, + { + rule_type = MIN_ROW + rule_value = 100 + } + ] + } } } From 7c70f043584d19103782f6d72821e7635b4f8e69 Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 28 Jun 2023 18:06:46 +0800 Subject: [PATCH 16/22] add ob in docs --- docs/en/connector-v2/sink/Jdbc.md | 1 + docs/en/connector-v2/source/Jdbc.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md index 43469df2905..f0dd52dbaa7 100644 --- a/docs/en/connector-v2/sink/Jdbc.md +++ b/docs/en/connector-v2/sink/Jdbc.md @@ -173,6 +173,7 @@ there are some reference value for params above. | Redshift | com.amazon.redshift.jdbc42.Driver | jdbc:redshift://localhost:5439/testdb | com.amazon.redshift.xa.RedshiftXADataSource | https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 | | Snowflake | net.snowflake.client.jdbc.SnowflakeDriver | jdbc:snowflake://.snowflakecomputing.com | / | https://mvnrepository.com/artifact/net.snowflake/snowflake-jdbc | | Vertica | com.vertica.jdbc.Driver | jdbc:vertica://localhost:5433 | / | https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/12.0.3-0/vertica-jdbc-12.0.3-0.jar | +| OceanBase | com.oceanbase.jdbc.Driver | jdbc:oceanbase://localhost:2881 | / | https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.3/oceanbase-client-2.4.3.jar | ## Example diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md index bb30abf91fa..7723f8f1590 100644 --- a/docs/en/connector-v2/source/Jdbc.md +++ b/docs/en/connector-v2/source/Jdbc.md @@ -125,6 +125,7 @@ there are some reference value for params above. | Snowflake | net.snowflake.client.jdbc.SnowflakeDriver | jdbc:snowflake://.snowflakecomputing.com | https://mvnrepository.com/artifact/net.snowflake/snowflake-jdbc | | Redshift | com.amazon.redshift.jdbc42.Driver | jdbc:redshift://localhost:5439/testdb?defaultRowFetchSize=1000 | https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 | | Vertica | com.vertica.jdbc.Driver | jdbc:vertica://localhost:5433 | https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/12.0.3-0/vertica-jdbc-12.0.3-0.jar | +| OceanBase | com.oceanbase.jdbc.Driver | jdbc:oceanbase://localhost:2881 | https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.3/oceanbase-client-2.4.3.jar | ## Example From 168b1b3001b5c31abea6207b5fb6b9219be3b3c5 Mon Sep 17 00:00:00 2001 From: He Wang Date: Thu, 29 Jun 2023 16:42:35 +0800 Subject: [PATCH 17/22] disable obmysql IT case --- .../connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java index 8297f1d3db0..f1f365931da 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; @@ -37,6 +38,7 @@ import java.util.ArrayList; import java.util.List; +@Disabled("Disabled due to insufficient hardware resources in the CI environment") public class JdbcOceanBaseMysqlIT extends JdbcOceanBaseITBase { @Override From 558e41bb12f5a463cf1eb656c7aeee5630407da4 Mon Sep 17 00:00:00 2001 From: He Wang Date: Mon, 3 Jul 2023 15:02:38 +0800 Subject: [PATCH 18/22] use compatible_mode instead of driver_type and add ob doc --- docs/en/connector-v2/sink/Jdbc.md | 6 +- docs/en/connector-v2/sink/OceanBase.md | 186 ++++++++++++++++++ docs/en/connector-v2/source/Jdbc.md | 6 +- docs/en/connector-v2/source/OceanBase.md | 168 ++++++++++++++++ .../jdbc/config/JdbcConnectionConfig.java | 16 +- .../seatunnel/jdbc/config/JdbcOptions.java | 8 +- .../jdbc/config/JdbcSourceConfig.java | 4 +- .../internal/dialect/JdbcDialectFactory.java | 4 +- .../internal/dialect/JdbcDialectLoader.java | 6 +- .../oceanbase/OceanBaseDialectFactory.java | 6 +- .../seatunnel/jdbc/sink/JdbcSink.java | 2 +- .../seatunnel/jdbc/sink/JdbcSinkFactory.java | 2 +- .../seatunnel/jdbc/source/JdbcSource.java | 2 +- .../jdbc/source/JdbcSourceFactory.java | 2 +- .../jdbc_oceanbase_mysql_source_and_sink.conf | 2 +- ...jdbc_oceanbase_oracle_source_and_sink.conf | 2 +- 16 files changed, 390 insertions(+), 32 deletions(-) create mode 100644 docs/en/connector-v2/sink/OceanBase.md create mode 100644 docs/en/connector-v2/source/OceanBase.md diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md index f0dd52dbaa7..f128f6b4b21 100644 --- a/docs/en/connector-v2/sink/Jdbc.md +++ b/docs/en/connector-v2/sink/Jdbc.md @@ -33,7 +33,7 @@ support `Xa transactions`. You can set `is_exactly_once=true` to enable it. | user | String | No | - | | password | String | No | - | | query | String | No | - | -| driver_type | String | No | - | +| compatible_mode | String | No | - | | database | String | No | - | | table | String | No | - | | primary_keys | Array | No | - | @@ -70,9 +70,9 @@ The URL of the JDBC connection. Refer to a case: jdbc:postgresql://localhost/tes Use this sql write upstream input datas to database. e.g `INSERT ...` -### driver_type [string] +### compatible_mode [string] -Use this field to represent the OceanBase driver. e.g 'mysql' +The compatible mode of database, required when the database supports multiple compatible modes. For example, when using OceanBase database, you need to set it to 'mysql' or 'oracle'. ### database [string] diff --git a/docs/en/connector-v2/sink/OceanBase.md b/docs/en/connector-v2/sink/OceanBase.md new file mode 100644 index 00000000000..054228b49b9 --- /dev/null +++ b/docs/en/connector-v2/sink/OceanBase.md @@ -0,0 +1,186 @@ +# OceanBase + +> JDBC OceanBase Sink Connector + +## Support Those Engines + +> Spark
+> Flink
+> Seatunnel Zeta
+ +## Key Features + +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [x] [cdc](../../concept/connector-v2-features.md) + +## Description + +Write data through jdbc. Support Batch mode and Streaming mode, support concurrent writing, support exactly-once semantics. + +## Supported DataSource Info + +| Datasource | Supported versions | Driver | Url | Maven | +|------------|--------------------------------|---------------------------|--------------------------------------|-------------------------------------------------------------------------------| +| OceanBase | All OceanBase server versions. | com.oceanbase.jdbc.Driver | jdbc:oceanbase://localhost:2883/test | [Download](https://mvnrepository.com/artifact/com.oceanbase/oceanbase-client) | + +## Database Dependency + +> Please download the support list corresponding to 'Maven' and copy it to the '$SEATNUNNEL_HOME/plugins/jdbc/lib/' working directory
+> For example: cp oceanbase-client-xxx.jar $SEATNUNNEL_HOME/plugins/jdbc/lib/ + +## Data Type Mapping + +### Mysql Mode + +| Mysql Data type | Seatunnel Data type | +|-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| BIT(1)
INT UNSIGNED | BOOLEAN | +| TINYINT
TINYINT UNSIGNED
SMALLINT
SMALLINT UNSIGNED
MEDIUMINT
MEDIUMINT UNSIGNED
INT
INTEGER
YEAR | INT | +| INT UNSIGNED
INTEGER UNSIGNED
BIGINT | BIGINT | +| BIGINT UNSIGNED | DECIMAL(20,0) | +| DECIMAL(x,y)(Get the designated column's specified column size.<38) | DECIMAL(x,y) | +| DECIMAL(x,y)(Get the designated column's specified column size.>38) | DECIMAL(38,18) | +| DECIMAL UNSIGNED | DECIMAL((Get the designated column's specified column size)+1,
(Gets the designated column's number of digits to right of the decimal point.))) | +| FLOAT
FLOAT UNSIGNED | FLOAT | +| DOUBLE
DOUBLE UNSIGNED | DOUBLE | +| CHAR
VARCHAR
TINYTEXT
MEDIUMTEXT
TEXT
LONGTEXT
JSON | STRING | +| DATE | DATE | +| TIME | TIME | +| DATETIME
TIMESTAMP | TIMESTAMP | +| TINYBLOB
MEDIUMBLOB
BLOB
LONGBLOB
BINARY
VARBINAR
BIT(n) | BYTES | +| GEOMETRY
UNKNOWN | Not supported yet | + +### Oracle Mode + +| Oracle Data type | Seatunnel Data type | +|-----------------------------------------------------------|---------------------| +| Number(p), p <= 9 | INT | +| Number(p), p <= 18 | BIGINT | +| Number(p), p > 18 | DECIMAL(38,18) | +| REAL
BINARY_FLOAT | FLOAT | +| BINARY_DOUBLE | DOUBLE | +| CHAR
NCHAR
NVARCHAR2
NCLOB
CLOB
ROWID | STRING | +| DATE | DATE | +| TIMESTAMP
TIMESTAMP WITH LOCAL TIME ZONE | TIMESTAMP | +| BLOB
RAW
LONG RAW
BFILE | BYTES | +| UNKNOWN | Not supported yet | + +## Sink Options + +| Name | Type | Required | Default | Description | +|-------------------------------------------|---------|----------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| url | String | Yes | - | The URL of the JDBC connection. Refer to a case: jdbc:oceanbase://localhost:2883/test | +| driver | String | Yes | - | The jdbc class name used to connect to the remote data source, should be `com.oceanbase.jdbc.Driver`. | +| user | String | No | - | Connection instance user name | +| password | String | No | - | Connection instance password | +| query | String | No | - | Use this sql write upstream input datas to database. e.g `INSERT ...`,`query` have the higher priority | +| compatible_mode | String | Yes | - | The compatible mode of OceanBase, can be 'mysql' or 'oracle'. | +| database | String | No | - | Use this `database` and `table-name` auto-generate sql and receive upstream input datas write to database.
This option is mutually exclusive with `query` and has a higher priority. | +| table | String | No | - | Use database and this table-name auto-generate sql and receive upstream input datas write to database.
This option is mutually exclusive with `query` and has a higher priority. | +| primary_keys | Array | No | - | This option is used to support operations such as `insert`, `delete`, and `update` when automatically generate sql. | +| support_upsert_by_query_primary_key_exist | Boolean | No | false | Choose to use INSERT sql, UPDATE sql to process update events(INSERT, UPDATE_AFTER) based on query primary key exists. This configuration is only used when database unsupport upsert syntax. **Note**: that this method has low performance | +| connection_check_timeout_sec | Int | No | 30 | The time in seconds to wait for the database operation used to validate the connection to complete. | +| max_retries | Int | No | 0 | The number of retries to submit failed (executeBatch) | +| batch_size | Int | No | 1000 | For batch writing, when the number of buffered records reaches the number of `batch_size` or the time reaches `batch_interval_ms`
, the data will be flushed into the database | +| batch_interval_ms | Int | No | 1000 | For batch writing, when the number of buffers reaches the number of `batch_size` or the time reaches `batch_interval_ms`, the data will be flushed into the database | +| generate_sink_sql | Boolean | No | false | Generate sql statements based on the database table you want to write to | +| max_commit_attempts | Int | No | 3 | The number of retries for transaction commit failures | +| transaction_timeout_sec | Int | No | -1 | The timeout after the transaction is opened, the default is -1 (never timeout). Note that setting the timeout may affect
exactly-once semantics | +| auto_commit | Boolean | No | true | Automatic transaction commit is enabled by default | +| common-options | | no | - | Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details | + +### Tips + +> If partition_column is not set, it will run in single concurrency, and if partition_column is set, it will be executed in parallel according to the concurrency of tasks. + +## Task Example + +### Simple: + +> This example defines a SeaTunnel synchronization task that automatically generates data through FakeSource and sends it to JDBC Sink. FakeSource generates a total of 16 rows of data (row.num=16), with each row having two fields, name (string type) and age (int type). The final target table is test_table will also be 16 rows of data in the table. Before run this job, you need create database test and table test_table in your mysql. And if you have not yet installed and deployed SeaTunnel, you need to follow the instructions in [Install SeaTunnel](../../start-v2/locally/deployment.md) to install and deploy SeaTunnel. And then follow the instructions in [Quick Start With SeaTunnel Engine](../../start-v2/locally/quick-start-seatunnel-engine.md) to run this job. + +``` +# Defining the runtime environment +env { + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + FakeSource { + parallelism = 1 + result_table_name = "fake" + row.num = 16 + schema = { + fields { + name = "string" + age = "int" + } + } + } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 +} + +transform { + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/category/transform-v2 +} + +sink { + jdbc { + url = "jdbc:oceanbase://localhost:2883/test" + driver = "com.oceanbase.jdbc.Driver" + user = "root" + password = "123456" + compatible_mode = "mysql" + query = "insert into test_table(name,age) values(?,?)" + } + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/category/sink-v2 +} +``` + +### Generate Sink SQL + +> This example not need to write complex sql statements, you can configure the database name table name to automatically generate add statements for you + +``` +sink { + jdbc { + url = "jdbc:oceanbase://localhost:2883/test" + driver = "com.oceanbase.jdbc.Driver" + user = "root" + password = "123456" + compatible_mode = "mysql" + # Automatically generate sql statements based on database table names + generate_sink_sql = true + database = test + table = test_table + } +} +``` + +### CDC(Change Data Capture) Event + +> CDC change data is also supported by us In this case, you need config database, table and primary_keys. + +``` +sink { + jdbc { + url = "jdbc:oceanbase://localhost:3306/test" + driver = "com.oceanbase.jdbc.Driver" + user = "root" + password = "123456" + compatible_mode = "mysql" + generate_sink_sql = true + # You need to configure both database and table + database = test + table = sink_table + primary_keys = ["id","name"] + } +} +``` + diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md index 7723f8f1590..a324316e594 100644 --- a/docs/en/connector-v2/source/Jdbc.md +++ b/docs/en/connector-v2/source/Jdbc.md @@ -35,7 +35,7 @@ supports query SQL and can achieve projection effect. | user | String | No | - | | password | String | No | - | | query | String | Yes | - | -| driver_type | String | Yes | - | +| compatible_mode | String | No | - | | connection_check_timeout_sec | Int | No | 30 | | partition_column | String | No | - | | partition_upper_bound | Long | No | - | @@ -64,9 +64,9 @@ The URL of the JDBC connection. Refer to a case: jdbc:postgresql://localhost/tes Query statement -### driver_type [string] +### compatible_mode [string] -Use this field to represent the OceanBase driver. e.g 'mysql' +The compatible mode of database, required when the database supports multiple compatible modes. For example, when using OceanBase database, you need to set it to 'mysql' or 'oracle'. ### connection_check_timeout_sec [int] diff --git a/docs/en/connector-v2/source/OceanBase.md b/docs/en/connector-v2/source/OceanBase.md new file mode 100644 index 00000000000..d237db2a22b --- /dev/null +++ b/docs/en/connector-v2/source/OceanBase.md @@ -0,0 +1,168 @@ +# OceanBase + +> JDBC OceanBase Source Connector + +## Support Those Engines + +> Spark
+> Flink
+> Seatunnel Zeta
+ +## Key Features + +- [x] [batch](../../concept/connector-v2-features.md) +- [ ] [stream](../../concept/connector-v2-features.md) +- [x] [exactly-once](../../concept/connector-v2-features.md) +- [x] [column projection](../../concept/connector-v2-features.md) +- [x] [parallelism](../../concept/connector-v2-features.md) +- [x] [support user-defined split](../../concept/connector-v2-features.md) + +## Description + +Read external data source data through JDBC. + +## Supported DataSource Info + +| Datasource | Supported versions | Driver | Url | Maven | +|------------|--------------------------------|---------------------------|--------------------------------------|-------------------------------------------------------------------------------| +| OceanBase | All OceanBase server versions. | com.oceanbase.jdbc.Driver | jdbc:oceanbase://localhost:2883/test | [Download](https://mvnrepository.com/artifact/com.oceanbase/oceanbase-client) | + +## Database Dependency + +> Please download the support list corresponding to 'Maven' and copy it to the '$SEATNUNNEL_HOME/plugins/jdbc/lib/' working directory
+> For example: cp oceanbase-client-xxx.jar $SEATNUNNEL_HOME/plugins/jdbc/lib/ + +## Data Type Mapping + +### Mysql Mode + +| Mysql Data type | Seatunnel Data type | +|-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| BIT(1)
INT UNSIGNED | BOOLEAN | +| TINYINT
TINYINT UNSIGNED
SMALLINT
SMALLINT UNSIGNED
MEDIUMINT
MEDIUMINT UNSIGNED
INT
INTEGER
YEAR | INT | +| INT UNSIGNED
INTEGER UNSIGNED
BIGINT | BIGINT | +| BIGINT UNSIGNED | DECIMAL(20,0) | +| DECIMAL(x,y)(Get the designated column's specified column size.<38) | DECIMAL(x,y) | +| DECIMAL(x,y)(Get the designated column's specified column size.>38) | DECIMAL(38,18) | +| DECIMAL UNSIGNED | DECIMAL((Get the designated column's specified column size)+1,
(Gets the designated column's number of digits to right of the decimal point.))) | +| FLOAT
FLOAT UNSIGNED | FLOAT | +| DOUBLE
DOUBLE UNSIGNED | DOUBLE | +| CHAR
VARCHAR
TINYTEXT
MEDIUMTEXT
TEXT
LONGTEXT
JSON | STRING | +| DATE | DATE | +| TIME | TIME | +| DATETIME
TIMESTAMP | TIMESTAMP | +| TINYBLOB
MEDIUMBLOB
BLOB
LONGBLOB
BINARY
VARBINAR
BIT(n) | BYTES | +| GEOMETRY
UNKNOWN | Not supported yet | + +### Oracle Mode + +| Oracle Data type | Seatunnel Data type | +|-----------------------------------------------------------|---------------------| +| Number(p), p <= 9 | INT | +| Number(p), p <= 18 | BIGINT | +| Number(p), p > 18 | DECIMAL(38,18) | +| REAL
BINARY_FLOAT | FLOAT | +| BINARY_DOUBLE | DOUBLE | +| CHAR
NCHAR
NVARCHAR2
NCLOB
CLOB
ROWID | STRING | +| DATE | DATE | +| TIMESTAMP
TIMESTAMP WITH LOCAL TIME ZONE | TIMESTAMP | +| BLOB
RAW
LONG RAW
BFILE | BYTES | +| UNKNOWN | Not supported yet | + +## Source Options + +| Name | Type | Required | Default | Description | +|------------------------------|--------|----------|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| url | String | Yes | - | The URL of the JDBC connection. Refer to a case: jdbc:oceanbase://localhost:2883/test | +| driver | String | Yes | - | The jdbc class name used to connect to the remote data source, should be `com.oceanbase.jdbc.Driver`. | +| user | String | No | - | Connection instance user name | +| password | String | No | - | Connection instance password | +| compatible_mode | String | Yes | - | The compatible mode of OceanBase, can be 'mysql' or 'oracle'. | +| query | String | Yes | - | Query statement | +| connection_check_timeout_sec | Int | No | 30 | The time in seconds to wait for the database operation used to validate the connection to complete | +| partition_column | String | No | - | The column name for parallelism's partition, only support numeric type primary key and only support single column. | +| partition_lower_bound | Long | No | - | The partition_column min value for scan, if not set SeaTunnel will query database get min value. | +| partition_upper_bound | Long | No | - | The partition_column max value for scan, if not set SeaTunnel will query database get max value. | +| partition_num | Int | No | job parallelism | The number of partition count, only support positive integer. Default value is job parallelism. | +| fetch_size | Int | No | 0 | For queries that return a large number of objects, you can configure
the row fetch size used in the query to improve performance by
reducing the number database hits required to satisfy the selection criteria.
Zero means use jdbc default value. | +| common-options | | No | - | Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details | + +### Tips + +> If partition_column is not set, it will run in single concurrency, and if partition_column is set, it will be executed in parallel according to the concurrency of tasks. + +## Task Example + +### Simple: + +``` +env { + execution.parallelism = 2 + job.mode = "BATCH" +} + +source { + Jdbc { + driver = "com.oceanbase.jdbc.Driver" + url = "jdbc:oceanbase://localhost:2883/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" + user = "root" + password = "" + compatible_mode = "mysql" + query = "select * from source" + } +} + +transform { + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql +} + +sink { + Console {} +} +``` + +### Parallel: + +> Read your query table in parallel with the shard field you configured and the shard data. You can do this if you want to read the whole table + +``` +source { + Jdbc { + driver = "com.oceanbase.jdbc.Driver" + url = "jdbc:oceanbase://localhost:2883/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" + user = "root" + password = "" + compatible_mode = "mysql" + query = "select * from source" + # Parallel sharding reads fields + partition_column = "id" + # Number of fragments + partition_num = 10 + } +} +``` + +### Parallel Boundary: + +> It is more efficient to read your data source according to the upper and lower boundaries you configured + +``` +source { + Jdbc { + driver = "com.oceanbase.jdbc.Driver" + url = "jdbc:oceanbase://localhost:2883/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" + user = "root" + password = "" + compatible_mode = "mysql" + query = "select * from source" + partition_column = "id" + partition_num = 10 + # Read start boundary + partition_lower_bound = 1 + # Read end boundary + partition_upper_bound = 500 + } +} +``` + diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java index db2366ae673..8fd427263e6 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcConnectionConfig.java @@ -27,7 +27,7 @@ public class JdbcConnectionConfig implements Serializable { public String url; public String driverName; - public String driverType; + public String compatibleMode; public int connectionCheckTimeoutSeconds = JdbcOptions.CONNECTION_CHECK_TIMEOUT_SEC.defaultValue(); public int maxRetries = JdbcOptions.MAX_RETRIES.defaultValue(); @@ -49,7 +49,7 @@ public class JdbcConnectionConfig implements Serializable { public static JdbcConnectionConfig of(ReadonlyConfig config) { JdbcConnectionConfig.Builder builder = JdbcConnectionConfig.builder(); builder.url(config.get(JdbcOptions.URL)); - builder.driverType(config.get(JdbcOptions.DRIVER_TYPE)); + builder.compatibleMode(config.get(JdbcOptions.COMPATIBLE_MODE)); builder.driverName(config.get(JdbcOptions.DRIVER)); builder.autoCommit(config.get(JdbcOptions.AUTO_COMMIT)); builder.maxRetries(config.get(JdbcOptions.MAX_RETRIES)); @@ -75,8 +75,8 @@ public String getDriverName() { return driverName; } - public String getDriverType() { - return driverType; + public String getCompatibleMode() { + return compatibleMode; } public boolean isAutoCommit() { @@ -126,7 +126,7 @@ public static JdbcConnectionConfig.Builder builder() { public static final class Builder { private String url; private String driverName; - private String driverType; + private String compatibleMode; private int connectionCheckTimeoutSeconds = JdbcOptions.CONNECTION_CHECK_TIMEOUT_SEC.defaultValue(); private int maxRetries = JdbcOptions.MAX_RETRIES.defaultValue(); @@ -152,8 +152,8 @@ public Builder driverName(String driverName) { return this; } - public Builder driverType(String driverType) { - this.driverType = driverType; + public Builder compatibleMode(String compatibleMode) { + this.compatibleMode = compatibleMode; return this; } @@ -217,7 +217,7 @@ public JdbcConnectionConfig build() { jdbcConnectionConfig.batchSize = this.batchSize; jdbcConnectionConfig.batchIntervalMs = this.batchIntervalMs; jdbcConnectionConfig.driverName = this.driverName; - jdbcConnectionConfig.driverType = this.driverType; + jdbcConnectionConfig.compatibleMode = this.compatibleMode; jdbcConnectionConfig.maxRetries = this.maxRetries; jdbcConnectionConfig.password = this.password; jdbcConnectionConfig.connectionCheckTimeoutSeconds = this.connectionCheckTimeoutSeconds; diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java index 77b32bc95ec..24ae0580f32 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcOptions.java @@ -36,8 +36,12 @@ public interface JdbcOptions { .intType() .defaultValue(30) .withDescription("connection check time second"); - Option DRIVER_TYPE = - Options.key("driver_type").stringType().noDefaultValue().withDescription("driver_type"); + Option COMPATIBLE_MODE = + Options.key("compatible_mode") + .stringType() + .noDefaultValue() + .withDescription( + "The compatible mode of database, required when the database supports multiple compatible modes. For example, when using OceanBase database, you need to set it to 'mysql' or 'oracle'."); Option MAX_RETRIES = Options.key("max_retries").intType().defaultValue(0).withDescription("max_retired"); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java index c748ff71951..00130b32acc 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java @@ -33,7 +33,7 @@ public class JdbcSourceConfig implements Serializable { private JdbcConnectionConfig jdbcConnectionConfig; public String query; - public String driverType; + public String compatibleMode; private String partitionColumn; private BigDecimal partitionUpperBound; private BigDecimal partitionLowerBound; @@ -45,7 +45,7 @@ public static JdbcSourceConfig of(ReadonlyConfig config) { builder.jdbcConnectionConfig(JdbcConnectionConfig.of(config)); builder.query(config.get(JdbcOptions.QUERY)); builder.fetchSize(config.get(JdbcOptions.FETCH_SIZE)); - config.getOptional(JdbcOptions.DRIVER_TYPE).ifPresent(builder::driverType); + config.getOptional(JdbcOptions.COMPATIBLE_MODE).ifPresent(builder::compatibleMode); config.getOptional(JdbcOptions.PARTITION_COLUMN).ifPresent(builder::partitionColumn); config.getOptional(JdbcOptions.PARTITION_UPPER_BOUND) .ifPresent(builder::partitionUpperBound); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java index 94b4f9bcd3c..3baa38e3eed 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java @@ -41,10 +41,10 @@ public interface JdbcDialectFactory { /** * Create a {@link JdbcDialect} instance based on the driver type. * - * @param driverType The driver type + * @param compatibleMode The compatible mode * @return a new instance of {@link JdbcDialect} */ - default JdbcDialect create(String driverType) { + default JdbcDialect create(String compatibleMode) { return create(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java index 9f4166ed021..b49df35ff3f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectLoader.java @@ -40,12 +40,12 @@ private JdbcDialectLoader() {} * Loads the unique JDBC Dialect that can handle the given database url. * * @param url A database URL. - * @param driverType The driver type. + * @param compatibleMode The compatible mode. * @throws IllegalStateException if the loader cannot find exactly one dialect that can * unambiguously process the given database URL. * @return The loaded dialect. */ - public static JdbcDialect load(String url, String driverType) { + public static JdbcDialect load(String url, String compatibleMode) { ClassLoader cl = Thread.currentThread().getContextClassLoader(); List foundFactories = discoverFactories(cl); @@ -90,7 +90,7 @@ public static JdbcDialect load(String url, String driverType) { .collect(Collectors.joining("\n")))); } - return matchingFactories.get(0).create(driverType); + return matchingFactories.get(0).create(compatibleMode); } private static List discoverFactories(ClassLoader classLoader) { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java index 836ece8601e..66df84205ed 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseDialectFactory.java @@ -36,12 +36,12 @@ public boolean acceptsURL(String url) { @Override public JdbcDialect create() { throw new UnsupportedOperationException( - "Can't create JdbcDialect without driver type for OceanBase"); + "Can't create JdbcDialect without compatible mode for OceanBase"); } @Override - public JdbcDialect create(@Nonnull String driverType) { - if ("oracle".equalsIgnoreCase(driverType)) { + public JdbcDialect create(@Nonnull String compatibleMode) { + if ("oracle".equalsIgnoreCase(compatibleMode)) { return new OracleDialect(); } return new MysqlDialect(); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java index 4527c14f5ff..4666eae1e51 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSink.java @@ -107,7 +107,7 @@ public void prepare(Config pluginConfig) throws PrepareFailException { this.dialect = JdbcDialectLoader.load( jdbcSinkConfig.getJdbcConnectionConfig().getUrl(), - jdbcSinkConfig.getJdbcConnectionConfig().getDriverType()); + jdbcSinkConfig.getJdbcConnectionConfig().getCompatibleMode()); this.dataSaveMode = DataSaveMode.KEEP_SCHEMA_AND_DATA; } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java index b738b364fe7..98fb2435129 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java @@ -85,7 +85,7 @@ public TableSink createSink(TableFactoryContext context) { JdbcDialect dialect = JdbcDialectLoader.load( sinkConfig.getJdbcConnectionConfig().getUrl(), - sinkConfig.getJdbcConnectionConfig().getDriverType()); + sinkConfig.getJdbcConnectionConfig().getCompatibleMode()); return () -> new JdbcSink( options, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java index a02cbc88605..732892b21d6 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSource.java @@ -101,7 +101,7 @@ public void prepare(Config pluginConfig) throws PrepareFailException { this.jdbcDialect = JdbcDialectLoader.load( jdbcSourceConfig.getJdbcConnectionConfig().getUrl(), - jdbcSourceConfig.getJdbcConnectionConfig().getDriverType()); + jdbcSourceConfig.getJdbcConnectionConfig().getCompatibleMode()); try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection()) { this.typeInfo = initTableField(connection); this.partitionParameter = diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java index dfd7164dc54..54900af4693 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java @@ -86,7 +86,7 @@ TableSource createSource(TableFactoryContext context) { JdbcDialect dialect = JdbcDialectLoader.load( config.getJdbcConnectionConfig().getUrl(), - config.getJdbcConnectionConfig().getDriverType()); + config.getJdbcConnectionConfig().getCompatibleMode()); TableSchema tableSchema = catalogTable.getTableSchema(); SeaTunnelRowType rowType = tableSchema.toPhysicalRowDataType(); Optional partitionParameter = diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf index 992d02556e7..b33ed611855 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf @@ -34,7 +34,7 @@ source { user = root password = "" query = "select * from source" - driver_type = "mysql" + compatible_mode = "mysql" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf index 3d85c1e5f98..53d6337198c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf @@ -34,7 +34,7 @@ source { user = "root" password = "" query = "SELECT * FROM source" - driver_type = "oracle" + compatible_mode = "oracle" } } From a19e2a28221c90f61c96c101be72b75959937706 Mon Sep 17 00:00:00 2001 From: He Wang Date: Fri, 7 Jul 2023 11:42:49 +0800 Subject: [PATCH 19/22] comments addressed --- docs/en/connector-v2/source/OceanBase.md | 2 +- .../connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java | 4 +++- .../connectors/seatunnel/jdbc/source/JdbcSourceFactory.java | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/en/connector-v2/source/OceanBase.md b/docs/en/connector-v2/source/OceanBase.md index d237db2a22b..f7fef990aeb 100644 --- a/docs/en/connector-v2/source/OceanBase.md +++ b/docs/en/connector-v2/source/OceanBase.md @@ -80,7 +80,7 @@ Read external data source data through JDBC. | compatible_mode | String | Yes | - | The compatible mode of OceanBase, can be 'mysql' or 'oracle'. | | query | String | Yes | - | Query statement | | connection_check_timeout_sec | Int | No | 30 | The time in seconds to wait for the database operation used to validate the connection to complete | -| partition_column | String | No | - | The column name for parallelism's partition, only support numeric type primary key and only support single column. | +| partition_column | String | No | - | The column name for parallelism's partition, only support numeric type column and string type column. | | partition_lower_bound | Long | No | - | The partition_column min value for scan, if not set SeaTunnel will query database get min value. | | partition_upper_bound | Long | No | - | The partition_column max value for scan, if not set SeaTunnel will query database get max value. | | partition_num | Int | No | job parallelism | The number of partition count, only support positive integer. Default value is job parallelism. | diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java index 98fb2435129..a9bb1c15554 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java @@ -41,6 +41,7 @@ import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.AUTO_COMMIT; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.BATCH_INTERVAL_MS; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.BATCH_SIZE; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.COMPATIBLE_MODE; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.CONNECTION_CHECK_TIMEOUT_SEC; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.DATABASE; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.DRIVER; @@ -109,7 +110,8 @@ public OptionRule optionRule() { GENERATE_SINK_SQL, AUTO_COMMIT, SUPPORT_UPSERT_BY_QUERY_PRIMARY_KEY_EXIST, - PRIMARY_KEYS) + PRIMARY_KEYS, + COMPATIBLE_MODE) .conditional( IS_EXACTLY_ONCE, true, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java index 54900af4693..43aa1c03d63 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Optional; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.COMPATIBLE_MODE; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.CONNECTION_CHECK_TIMEOUT_SEC; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.DRIVER; import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcOptions.FETCH_SIZE; @@ -231,7 +232,8 @@ public OptionRule optionRule() { PARTITION_COLUMN, PARTITION_UPPER_BOUND, PARTITION_LOWER_BOUND, - PARTITION_NUM) + PARTITION_NUM, + COMPATIBLE_MODE) .build(); } From 6a4ae96ddd524a53fb94d9684bb1b5b6f699e09f Mon Sep 17 00:00:00 2001 From: He Wang Date: Fri, 7 Jul 2023 14:24:24 +0800 Subject: [PATCH 20/22] add compare result in ut --- .../seatunnel/jdbc/JdbcOceanBaseITBase.java | 48 ++++++++- .../seatunnel/jdbc/JdbcOceanBaseMysqlIT.java | 100 +++++++++--------- .../seatunnel/jdbc/JdbcOceanBaseOracleIT.java | 34 +++--- .../jdbc_oceanbase_mysql_source_and_sink.conf | 27 ++--- ...jdbc_oceanbase_oracle_source_and_sink.conf | 37 +++---- 5 files changed, 143 insertions(+), 103 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java index ceac9908b4b..b8202e697a1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java @@ -22,9 +22,17 @@ import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Assertions; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; public abstract class JdbcOceanBaseITBase extends AbstractJdbcIT { @@ -49,6 +57,8 @@ public abstract class JdbcOceanBaseITBase extends AbstractJdbcIT { abstract String createSqlTemplate(); + abstract String[] getFieldNames(); + @Override JdbcCase getJdbcCase() { Map containerEnv = new HashMap<>(); @@ -81,7 +91,43 @@ JdbcCase getJdbcCase() { } @Override - void compareResult() {} + void compareResult() { + String sourceSql = + String.format( + "select * from %s.%s order by 1", OCEANBASE_DATABASE, OCEANBASE_SOURCE); + String sinkSql = + String.format("select * from %s.%s order by 1", OCEANBASE_DATABASE, OCEANBASE_SINK); + try { + Statement sourceStatement = connection.createStatement(); + Statement sinkStatement = connection.createStatement(); + ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); + ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql); + Assertions.assertEquals( + sourceResultSet.getMetaData().getColumnCount(), + sinkResultSet.getMetaData().getColumnCount()); + while (sourceResultSet.next()) { + if (sinkResultSet.next()) { + for (String column : getFieldNames()) { + Object source = sourceResultSet.getObject(column); + Object sink = sinkResultSet.getObject(column); + if (!Objects.deepEquals(source, sink)) { + InputStream sourceAsciiStream = sourceResultSet.getBinaryStream(column); + InputStream sinkAsciiStream = sinkResultSet.getBinaryStream(column); + String sourceValue = + IOUtils.toString(sourceAsciiStream, StandardCharsets.UTF_8); + String sinkValue = + IOUtils.toString(sinkAsciiStream, StandardCharsets.UTF_8); + Assertions.assertEquals(sourceValue, sinkValue); + } + } + } + } + sourceResultSet.last(); + sinkResultSet.last(); + } catch (Exception e) { + throw new RuntimeException("Compare result error", e); + } + } @Override String driverUrl() { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java index f1f365931da..548fecaee66 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseMysqlIT.java @@ -120,57 +120,61 @@ String createSqlTemplate() { + " `c_decimal_unsigned_30` DECIMAL(30) unsigned DEFAULT NULL,\n" + " `c_decimal_30` DECIMAL(30) DEFAULT NULL\n" + ");"; - }; + } + + @Override + String[] getFieldNames() { + return new String[] { + "c_bit_1", + "c_bit_8", + "c_bit_16", + "c_bit_32", + "c_bit_64", + "c_boolean", + "c_tinyint", + "c_tinyint_unsigned", + "c_smallint", + "c_smallint_unsigned", + "c_mediumint", + "c_mediumint_unsigned", + "c_int", + "c_integer", + "c_year", + "c_int_unsigned", + "c_integer_unsigned", + "c_bigint", + "c_bigint_unsigned", + "c_decimal", + "c_decimal_unsigned", + "c_float", + "c_float_unsigned", + "c_double", + "c_double_unsigned", + "c_char", + "c_tinytext", + "c_mediumtext", + "c_text", + "c_varchar", + "c_json", + "c_longtext", + "c_date", + "c_datetime", + "c_timestamp", + "c_tinyblob", + "c_mediumblob", + "c_blob", + "c_longblob", + "c_varbinary", + "c_binary", + "c_bigint_30", + "c_decimal_unsigned_30", + "c_decimal_30", + }; + } @Override Pair> initTestData() { - String[] fieldNames = - new String[] { - "c_bit_1", - "c_bit_8", - "c_bit_16", - "c_bit_32", - "c_bit_64", - "c_boolean", - "c_tinyint", - "c_tinyint_unsigned", - "c_smallint", - "c_smallint_unsigned", - "c_mediumint", - "c_mediumint_unsigned", - "c_int", - "c_integer", - "c_year", - "c_int_unsigned", - "c_integer_unsigned", - "c_bigint", - "c_bigint_unsigned", - "c_decimal", - "c_decimal_unsigned", - "c_float", - "c_float_unsigned", - "c_double", - "c_double_unsigned", - "c_char", - "c_tinytext", - "c_mediumtext", - "c_text", - "c_varchar", - "c_json", - "c_longtext", - "c_date", - "c_datetime", - "c_timestamp", - "c_tinyblob", - "c_mediumblob", - "c_blob", - "c_longblob", - "c_varbinary", - "c_binary", - "c_bigint_30", - "c_decimal_unsigned_30", - "c_decimal_30", - }; + String[] fieldNames = getFieldNames(); List rows = new ArrayList<>(); BigDecimal bigintValue = new BigDecimal("2844674407371055000"); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java index 56a065e13d1..4c3cca5ddc1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseOracleIT.java @@ -113,23 +113,27 @@ String createSqlTemplate() { + ")"; } + @Override + String[] getFieldNames() { + return new String[] { + "VARCHAR_10_COL", + "CHAR_10_COL", + "CLOB_COL", + "NUMBER_3_SF_2_DP", + "INTEGER_COL", + "FLOAT_COL", + "REAL_COL", + "BINARY_FLOAT_COL", + "BINARY_DOUBLE_COL", + "DATE_COL", + "TIMESTAMP_WITH_3_FRAC_SEC_COL", + "TIMESTAMP_WITH_LOCAL_TZ" + }; + } + @Override Pair> initTestData() { - String[] fieldNames = - new String[] { - "VARCHAR_10_COL", - "CHAR_10_COL", - "CLOB_COL", - "NUMBER_3_SF_2_DP", - "INTEGER_COL", - "FLOAT_COL", - "REAL_COL", - "BINARY_FLOAT_COL", - "BINARY_DOUBLE_COL", - "DATE_COL", - "TIMESTAMP_WITH_3_FRAC_SEC_COL", - "TIMESTAMP_WITH_LOCAL_TZ" - }; + String[] fieldNames = getFieldNames(); List rows = new ArrayList<>(); for (int i = 0; i < 100; i++) { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf index b33ed611855..098d3ffae26 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf @@ -30,10 +30,10 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** Jdbc { driver = com.oceanbase.jdbc.Driver - url = "jdbc:oceanbase://e2e_oceanbase_mysql:2881/seatunnel?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" + url = "jdbc:oceanbase://e2e_oceanbase_mysql:2881/seatunnel?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true&serverTimezone=UTC" user = root password = "" - query = "select * from source" + query = "SELECT c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, c_mediumint, c_mediumint_unsigned, c_int, c_integer, c_bigint, c_bigint_unsigned, c_decimal, c_decimal_unsigned, c_float, c_float_unsigned, c_double, c_double_unsigned, c_char, c_tinytext, c_mediumtext, c_text, c_varchar, c_json, c_longtext, c_date, c_datetime, c_timestamp, c_tinyblob, c_mediumblob, c_blob, c_longblob, c_varbinary, c_binary, c_year, c_int_unsigned, c_integer_unsigned, c_bigint_30, c_decimal_unsigned_30, c_decimal_30 FROM source" compatible_mode = "mysql" } @@ -42,21 +42,14 @@ source { } sink { - Assert { - rules = - { - row_rules = [ - { - rule_type = MAX_ROW - rule_value = 100 - }, - { - rule_type = MIN_ROW - rule_value = 100 - } - ] - } + Jdbc { + driver = com.oceanbase.jdbc.Driver + url = "jdbc:oceanbase://e2e_oceanbase_mysql:2881/seatunnel?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true&serverTimezone=UTC" + user = root + password = "" + query = "insert into sink(c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, c_mediumint, c_mediumint_unsigned, c_int, c_integer, c_bigint, c_bigint_unsigned, c_decimal, c_decimal_unsigned, c_float, c_float_unsigned, c_double, c_double_unsigned, c_char, c_tinytext, c_mediumtext, c_text, c_varchar, c_json, c_longtext, c_date, c_datetime, c_timestamp, c_tinyblob, c_mediumblob, c_blob, c_longblob, c_varbinary, c_binary, c_year, c_int_unsigned, c_integer_unsigned,c_bigint_30,c_decimal_unsigned_30,c_decimal_30) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" + compatible_mode = "mysql" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert + # please go to https://seatunnel.apache.org/docs/connector-v2/sink } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf index 53d6337198c..bf2b1ccf067 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_oracle_source_and_sink.conf @@ -27,34 +27,27 @@ env { } source { - jdbc{ + jdbc{ # This is a example source plugin **only for test and demonstrate the feature source plugin** - url = "jdbc:oceanbase://e2e_oceanbase_oracle:2883/seatunnel" - driver = com.oceanbase.jdbc.Driver - user = "root" - password = "" - query = "SELECT * FROM source" - compatible_mode = "oracle" - } + url = "jdbc:oceanbase://e2e_oceanbase_oracle:2883/seatunnel" + driver = com.oceanbase.jdbc.Driver + user = "root" + password = "" + query = "SELECT VARCHAR_10_COL,CHAR_10_COL,CLOB_COL,NUMBER_3_SF_2_DP,INTEGER_COL,FLOAT_COL,REAL_COL,BINARY_FLOAT_COL,BINARY_DOUBLE_COL,DATE_COL,TIMESTAMP_WITH_3_FRAC_SEC_COL,TIMESTAMP_WITH_LOCAL_TZ FROM source" + compatible_mode = "oracle" + } } transform { } sink { - Assert { - rules = - { - row_rules = [ - { - rule_type = MAX_ROW - rule_value = 100 - }, - { - rule_type = MIN_ROW - rule_value = 100 - } - ] - } + jdbc{ + url = "jdbc:oceanbase://e2e_oceanbase_oracle:2883/seatunnel" + driver = com.oceanbase.jdbc.Driver + user = "root" + password = "" + query = "INSERT INTO sink (VARCHAR_10_COL,CHAR_10_COL,CLOB_COL,NUMBER_3_SF_2_DP,INTEGER_COL,FLOAT_COL,REAL_COL,BINARY_FLOAT_COL,BINARY_DOUBLE_COL,DATE_COL,TIMESTAMP_WITH_3_FRAC_SEC_COL,TIMESTAMP_WITH_LOCAL_TZ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)" + compatible_mode = "oracle" } } From 465d9e5ffb95f8f4bfe04cc894e3d90ae859619e Mon Sep 17 00:00:00 2001 From: He Wang Date: Fri, 7 Jul 2023 17:57:41 +0800 Subject: [PATCH 21/22] set pulsar container timeout to 3 min --- .../apache/seatunnel/e2e/connector/pulsar/PulsarBatchIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarBatchIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarBatchIT.java index b1ea69efac6..092f37f9bc4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarBatchIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarBatchIT.java @@ -56,6 +56,7 @@ import lombok.extern.slf4j.Slf4j; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -113,6 +114,7 @@ public void startUp() throws Exception { new PulsarContainer(DockerImageName.parse(PULSAR_IMAGE_NAME)) .withNetwork(NETWORK) .withNetworkAliases(PULSAR_HOST) + .withStartupTimeout(Duration.ofMinutes(3)) .withLogConsumer( new Slf4jLogConsumer( DockerLoggerFactory.getLogger(PULSAR_IMAGE_NAME))); From 854e9a988d4b921bab628bdc989d868c0b3e32b6 Mon Sep 17 00:00:00 2001 From: He Wang Date: Mon, 10 Jul 2023 14:28:40 +0800 Subject: [PATCH 22/22] fix comments --- docs/en/connector-v2/sink/OceanBase.md | 6 +++--- docs/en/connector-v2/source/OceanBase.md | 6 +++--- .../seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/connector-v2/sink/OceanBase.md b/docs/en/connector-v2/sink/OceanBase.md index 054228b49b9..ec87ce3d36d 100644 --- a/docs/en/connector-v2/sink/OceanBase.md +++ b/docs/en/connector-v2/sink/OceanBase.md @@ -6,7 +6,7 @@ > Spark
> Flink
-> Seatunnel Zeta
+> SeaTunnel Zeta
## Key Features @@ -32,7 +32,7 @@ Write data through jdbc. Support Batch mode and Streaming mode, support concurre ### Mysql Mode -| Mysql Data type | Seatunnel Data type | +| Mysql Data type | SeaTunnel Data type | |-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| | BIT(1)
INT UNSIGNED | BOOLEAN | | TINYINT
TINYINT UNSIGNED
SMALLINT
SMALLINT UNSIGNED
MEDIUMINT
MEDIUMINT UNSIGNED
INT
INTEGER
YEAR | INT | @@ -52,7 +52,7 @@ Write data through jdbc. Support Batch mode and Streaming mode, support concurre ### Oracle Mode -| Oracle Data type | Seatunnel Data type | +| Oracle Data type | SeaTunnel Data type | |-----------------------------------------------------------|---------------------| | Number(p), p <= 9 | INT | | Number(p), p <= 18 | BIGINT | diff --git a/docs/en/connector-v2/source/OceanBase.md b/docs/en/connector-v2/source/OceanBase.md index f7fef990aeb..9625ef4fbb9 100644 --- a/docs/en/connector-v2/source/OceanBase.md +++ b/docs/en/connector-v2/source/OceanBase.md @@ -6,7 +6,7 @@ > Spark
> Flink
-> Seatunnel Zeta
+> SeaTunnel Zeta
## Key Features @@ -36,7 +36,7 @@ Read external data source data through JDBC. ### Mysql Mode -| Mysql Data type | Seatunnel Data type | +| Mysql Data type | SeaTunnel Data type | |-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| | BIT(1)
INT UNSIGNED | BOOLEAN | | TINYINT
TINYINT UNSIGNED
SMALLINT
SMALLINT UNSIGNED
MEDIUMINT
MEDIUMINT UNSIGNED
INT
INTEGER
YEAR | INT | @@ -56,7 +56,7 @@ Read external data source data through JDBC. ### Oracle Mode -| Oracle Data type | Seatunnel Data type | +| Oracle Data type | SeaTunnel Data type | |-----------------------------------------------------------|---------------------| | Number(p), p <= 9 | INT | | Number(p), p <= 18 | BIGINT | diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java index 3baa38e3eed..3d66de65909 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialectFactory.java @@ -39,7 +39,7 @@ public interface JdbcDialectFactory { JdbcDialect create(); /** - * Create a {@link JdbcDialect} instance based on the driver type. + * Create a {@link JdbcDialect} instance based on the driver type and compatible mode. * * @param compatibleMode The compatible mode * @return a new instance of {@link JdbcDialect}