Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](ctas) fix NPE when ctas with old planner and varchar issue (#39744) #39871

Merged
merged 1 commit into from
Aug 24, 2024
Merged

[fix](ctas) fix NPE when ctas with old planner and varchar issue (#39744) #39871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1307,8 +1307,9 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio
if (resultExpr.getSrcSlotRef() != null
&& resultExpr.getSrcSlotRef().getTable() != null
&& !resultExpr.getSrcSlotRef().getTable().isManagedTable()) {
if (createTableStmt.getPartitionDesc().inIdentifierPartitions(
resultExpr.getSrcSlotRef().getColumnName())
if ((createTableStmt.getPartitionDesc() != null
&& createTableStmt.getPartitionDesc().inIdentifierPartitions(
resultExpr.getSrcSlotRef().getColumnName()))
|| (createTableStmt.getDistributionDesc() != null
&& createTableStmt.getDistributionDesc().inDistributionColumns(
resultExpr.getSrcSlotRef().getColumnName()))) {
Expand Down
17 changes: 14 additions & 3 deletions ...-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
Slot s = slots.get(i);
DataType dataType = s.getDataType().conversion();
if (i == 0 && dataType.isStringType()) {
// first column of olap table can not be string type.
// So change it to varchar type.
dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH);
} else {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
Expand All @@ -135,13 +137,21 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
if (createTableInfo.getPartitionTableInfo().inIdentifierPartitions(s.getName())
|| (createTableInfo.getDistribution() != null
&& createTableInfo.getDistribution().inDistributionColumns(s.getName()))) {
// String type can not be used in partition/distributed column
// String type can not be used in partition/distributed column,
// so we replace it to varchar
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, VarcharType.MAX_VARCHAR_TYPE);
} else {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, StringType.INSTANCE);
if (i == 0) {
// first column of olap table can not be string type.
// So change it to varchar type.
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, VarcharType.MAX_VARCHAR_TYPE);
} else {
// change varchar/char column from external table to string type
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, StringType.INSTANCE);
}
}
}
} else {
Expand Down Expand Up @@ -212,3 +222,4 @@ public CreateTableInfo getCreateTableInfo() {
return createTableInfo;
}
}

18 changes: 18 additions & 0 deletions regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,3 +433,21 @@ doris

-- !sql --

-- !sql --
int_u bigint Yes true \N
text varchar(65533) Yes true \N
t2 text Yes false \N NONE

-- !sql --
varchar varchar(65533) Yes true \N
int_u bigint Yes false \N NONE

-- !sql --
int_u bigint Yes true \N
text varchar(65533) Yes true \N
t2 varchar(65533) Yes false \N NONE

-- !sql --
varchar varchar(65533) Yes true \N
int_u bigint Yes false \N NONE

25 changes: 25 additions & 0 deletions regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar"
// String driver_url = "mysql-connector-java-8.0.25.jar"
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String user = "test_jdbc_user";
String pwd = '123456';
Expand Down Expand Up @@ -614,6 +615,30 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc

order_qt_sql """select * from mysql_conjuncts.doris_test.text_push where pk <=7;"""

// test create table as select
sql """use internal.${internal_db_name}"""
sql """drop table if exists ctas_partition_text_1"""
sql """drop table if exists ctas_partition_text_2"""
sql """drop table if exists ctas_partition_text_3"""
sql """drop table if exists ctas_partition_text_4"""
sql """set enable_nereids_planner=true"""
// 1. test text type column as distribution col
sql """create table ctas_partition_text_1 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_1"""
// 2. test varchar type column as first col
sql """create table ctas_partition_text_2 distributed by hash(int_u) buckets 1 properties("replication_num" = "1") as select varchar, int_u from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_2"""
// ctas logic is different between new and old planner.
// so need to test both.
// the old planner's test can be removed once the old planner is removed.
sql """set enable_nereids_planner=false"""
// 1. test text type column as distribution col
sql """create table ctas_partition_text_3 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_3"""
// 2. test varchar type column as first col
sql """create table ctas_partition_text_4 distributed by hash(int_u) buckets 1 properties("replication_num" = "1") as select varchar, int_u from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_4"""

sql """drop catalog if exists mysql_conjuncts;"""
}
}
Expand Down
Loading