Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[branch-2.1][improvement](jdbc catalog) Compatible with ojdbc6 by adding version check #39408

Merged
merged 1 commit into from
Aug 17, 2024
Merged

[branch-2.1][improvement](jdbc catalog) Compatible with ojdbc6 by adding version check #39408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions fe/be-java-extensions/jdbc-scanner/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ under the License.
<artifactId>HikariCP</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.semver4j</groupId>
<artifactId>semver4j</artifactId>
</dependency>
</dependencies>

<build>
Expand Down
30 changes: 29 additions & 1 deletion fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.semver4j.Semver;

import java.io.FileNotFoundException;
import java.lang.reflect.Array;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
Expand Down Expand Up @@ -65,6 +67,7 @@ public abstract class BaseJdbcExecutor implements JdbcExecutor {
protected VectorTable outputTable = null;
protected int batchSizeNum = 0;
protected int curBlockRows = 0;
protected String jdbcDriverVersion;

public BaseJdbcExecutor(byte[] thriftParams) throws Exception {
TJdbcExecutorCtorParams request = new TJdbcExecutorCtorParams();
Expand All @@ -91,11 +94,12 @@ public BaseJdbcExecutor(byte[] thriftParams) throws Exception {
.setConnectionPoolKeepAlive(request.connection_pool_keep_alive);
JdbcDataSource.getDataSource().setCleanupInterval(request.connection_pool_cache_clear_time);
init(config, request.statement);
this.jdbcDriverVersion = getJdbcDriverVersion();
}

public void close() throws Exception {
try {
if (stmt != null) {
if (stmt != null && !stmt.isClosed()) {
try {
stmt.cancel();
} catch (SQLException e) {
Expand Down Expand Up @@ -524,6 +528,30 @@ private void insertNullColumn(int parameterIndex, ColumnType.Type dorisType)
}
}

private String getJdbcDriverVersion() {
try {
if (conn != null) {
DatabaseMetaData metaData = conn.getMetaData();
return metaData.getDriverVersion();
} else {
return null;
}
} catch (SQLException e) {
LOG.warn("Failed to retrieve JDBC Driver version", e);
return null;
}
}

protected boolean isJdbcVersionGreaterThanOrEqualTo(String version) {
Semver currentVersion = Semver.coerce(jdbcDriverVersion);
Semver targetVersion = Semver.coerce(version);
if (currentVersion != null && targetVersion != null) {
return currentVersion.isGreaterThanOrEqualTo(targetVersion);
} else {
return false;
}
}

protected String trimSpaces(String str) {
int end = str.length() - 1;
while (end >= 0 && str.charAt(end) == ' ') {
Expand Down
115 changes: 78 additions & 37 deletions ...-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import java.nio.charset.StandardCharsets;
import java.sql.Clob;
import java.sql.SQLException;
import java.time.LocalDate;
import java.sql.Timestamp;
import java.time.LocalDateTime;

public class OracleJdbcExecutor extends BaseJdbcExecutor {
Expand Down Expand Up @@ -65,42 +65,83 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int

@Override
protected Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
try {
switch (type.getType()) {
case TINYINT:
return resultSet.getObject(columnIndex + 1, Byte.class);
case SMALLINT:
return resultSet.getObject(columnIndex + 1, Short.class);
case INT:
return resultSet.getObject(columnIndex + 1, Integer.class);
case BIGINT:
return resultSet.getObject(columnIndex + 1, Long.class);
case FLOAT:
return resultSet.getObject(columnIndex + 1, Float.class);
case DOUBLE:
return resultSet.getObject(columnIndex + 1, Double.class);
case LARGEINT:
case DECIMALV2:
case DECIMAL32:
case DECIMAL64:
case DECIMAL128:
return resultSet.getObject(columnIndex + 1, BigDecimal.class);
case DATE:
case DATEV2:
return resultSet.getObject(columnIndex + 1, LocalDate.class);
case DATETIME:
case DATETIMEV2:
return resultSet.getObject(columnIndex + 1, LocalDateTime.class);
case CHAR:
case VARCHAR:
case STRING:
return resultSet.getObject(columnIndex + 1);
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
} catch (AbstractMethodError e) {
LOG.warn("Detected an outdated ojdbc driver. Please use ojdbc8 or above.", e);
throw new SQLException("Detected an outdated ojdbc driver. Please use ojdbc8 or above.");
if (isJdbcVersionGreaterThanOrEqualTo("12.2.0")) {
return newGetColumnValue(columnIndex, type, replaceStringList);
} else {
return oldGetColumnValue(columnIndex, type, replaceStringList);
}
}

private Object newGetColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
switch (type.getType()) {
case TINYINT:
return resultSet.getObject(columnIndex + 1, Byte.class);
case SMALLINT:
return resultSet.getObject(columnIndex + 1, Short.class);
case INT:
return resultSet.getObject(columnIndex + 1, Integer.class);
case BIGINT:
return resultSet.getObject(columnIndex + 1, Long.class);
case FLOAT:
return resultSet.getObject(columnIndex + 1, Float.class);
case DOUBLE:
return resultSet.getObject(columnIndex + 1, Double.class);
case LARGEINT:
case DECIMALV2:
case DECIMAL32:
case DECIMAL64:
case DECIMAL128:
return resultSet.getObject(columnIndex + 1, BigDecimal.class);
case DATETIME:
case DATETIMEV2:
return resultSet.getObject(columnIndex + 1, LocalDateTime.class);
case CHAR:
case VARCHAR:
case STRING:
return resultSet.getObject(columnIndex + 1);
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
}

private Object oldGetColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
switch (type.getType()) {
case TINYINT:
byte tinyIntVal = resultSet.getByte(columnIndex + 1);
return resultSet.wasNull() ? null : tinyIntVal;
case SMALLINT:
short smallIntVal = resultSet.getShort(columnIndex + 1);
return resultSet.wasNull() ? null : smallIntVal;
case INT:
int intVal = resultSet.getInt(columnIndex + 1);
return resultSet.wasNull() ? null : intVal;
case BIGINT:
long bigIntVal = resultSet.getLong(columnIndex + 1);
return resultSet.wasNull() ? null : bigIntVal;
case FLOAT:
float floatVal = resultSet.getFloat(columnIndex + 1);
return resultSet.wasNull() ? null : floatVal;
case DOUBLE:
double doubleVal = resultSet.getDouble(columnIndex + 1);
return resultSet.wasNull() ? null : doubleVal;
case LARGEINT:
case DECIMALV2:
case DECIMAL32:
case DECIMAL64:
case DECIMAL128:
BigDecimal decimalVal = resultSet.getBigDecimal(columnIndex + 1);
return resultSet.wasNull() ? null : decimalVal;
case DATETIME:
case DATETIMEV2:
Timestamp timestampVal = resultSet.getTimestamp(columnIndex + 1);
return resultSet.wasNull() ? null : timestampVal.toLocalDateTime();
case CHAR:
case VARCHAR:
case STRING:
Object stringVal = resultSet.getObject(columnIndex + 1);
return resultSet.wasNull() ? null : stringVal;
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
}

Expand Down
6 changes: 6 additions & 0 deletions fe/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,7 @@ under the License.
<flatbuffers.version>1.12.0</flatbuffers.version>
<jacoco.version>0.8.10</jacoco.version>
<airlift.version>202</airlift.version>
<semver4j.version>5.3.0</semver4j.version>
</properties>
<profiles>
<profile>
Expand Down Expand Up @@ -1671,6 +1672,11 @@ under the License.
<artifactId>concurrent</artifactId>
<version>${airlift.version}</version>
</dependency>
<dependency>
<groupId>org.semver4j</groupId>
<artifactId>semver4j</artifactId>
<version>${semver4j.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
Expand Down
4 changes: 4 additions & 0 deletions regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,3 +265,7 @@ Doris
-- !query_lower_3 --
doris

-- !query_ojdbc6_all_types --
1 111 123 7456123.89 573 34 673.43 34.1264 60.0 23.231 99 9999 999999999 999999999999999999 999 99999 9999999999 9999999999999999999 1 china beijing alice abcdefghrjkmnopq 123.45 12300 0.0012345 2022-01-21T05:23:01 2019-11-12T20:33:57.999 2019-11-12T20:33:57.999998 2019-11-12T20:33:57.999996 2019-11-12T20:33:57.999997 223-9 12 10:23:1.123457000
2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N

16 changes: 16 additions & 0 deletions regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/ojdbc8.jar"
String driver6_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/ojdbc6.jar"
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String catalog_name = "oracle_catalog";
String internal_db_name = "regression_test_jdbc_catalog_p0";
Expand Down Expand Up @@ -281,5 +282,20 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d
qt_query_lower_3 """ select doris_3 from doris_test.lower_test; """

sql """drop catalog if exists ${catalog_name} """

// test for ojdbc6
sql """drop catalog if exists oracle_ojdbc6; """
sql """create catalog if not exists oracle_ojdbc6 properties(
"type"="jdbc",
"user"="doris_test",
"password"="123456",
"jdbc_url" = "jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}",
"driver_url" = "${driver6_url}",
"driver_class" = "oracle.jdbc.OracleDriver"
);"""
sql """ use oracle_ojdbc6.DORIS_TEST; """
qt_query_ojdbc6_all_types """ select * from oracle_ojdbc6.DORIS_TEST.TEST_ALL_TYPES order by 1; """

sql """drop catalog if exists oracle_ojdbc6; """
}
}
Loading