Skip to content

Commit

Permalink
[Fix](mutli-catalog) Use decimal v3 type to fix decimal loss issue in…
Browse files Browse the repository at this point in the history 
… multi-catalog module. (apache#18835)

Fix decimal v3 precision loss issues in the multi-catalog module.
Now it will use decimal v3 to represent decimal type in the multi-catalog module.
Regression Test: `test_load_with_decimal.groovy`
  • Loading branch information
@kaka11chen @xiaokang
kaka11chen authored and xiaokang committed Apr 20, 2023
1 parent ce13274 commit 2929099
Show file tree
Hide file tree
Showing 18 changed files with 2,918 additions and 2,756 deletions.
18 changes: 18 additions & 0 deletions be/src/runtime/types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,24 @@ struct TypeDescriptor {
return ret;
}

static TypeDescriptor create_decimalv3_type(int precision, int scale) {
DCHECK_LE(precision, MAX_PRECISION);
DCHECK_LE(scale, MAX_SCALE);
DCHECK_GE(precision, 0);
DCHECK_LE(scale, precision);
TypeDescriptor ret;
if (precision <= MAX_DECIMAL4_PRECISION) {
ret.type = TYPE_DECIMAL32;
} else if (precision <= MAX_DECIMAL8_PRECISION) {
ret.type = TYPE_DECIMAL64;
} else {
ret.type = TYPE_DECIMAL128I;
}
ret.precision = precision;
ret.scale = scale;
return ret;
}

static TypeDescriptor from_thrift(const TTypeDesc& t) {
int idx = 0;
TypeDescriptor result(t.types, &idx);
Expand Down
5 changes: 4 additions & 1 deletion be/src/vec/exec/format/json/new_json_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -878,8 +878,11 @@ Status NewJsonReader::_write_data_to_column(rapidjson::Value::ConstValueIterator
wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, value->GetUint64());
} else if (value->IsInt64()) {
wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, value->GetInt64());
} else if (value->IsFloat() || value->IsDouble()) {
auto end = fmt::format_to(tmp_buf, "{}", value->GetDouble());
wbytes = end - tmp_buf;
} else {
wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%f", value->GetDouble());
return Status::InternalError("It should not here.");
}
str_value = tmp_buf;
break;
Expand Down
4 changes: 2 additions & 2 deletions be/src/vec/exec/format/orc/vorc_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -580,8 +580,8 @@ TypeDescriptor OrcReader::_convert_to_doris_type(const orc::Type* orc_type) {
case orc::TypeKind::TIMESTAMP:
return TypeDescriptor(PrimitiveType::TYPE_DATETIMEV2);
case orc::TypeKind::DECIMAL:
// TODO: using decimal v3 instead
return TypeDescriptor(PrimitiveType::TYPE_DECIMALV2);
return TypeDescriptor::create_decimalv3_type(orc_type->getPrecision(),
orc_type->getScale());
case orc::TypeKind::DATE:
return TypeDescriptor(PrimitiveType::TYPE_DATEV2);
case orc::TypeKind::VARCHAR:
Expand Down
15 changes: 9 additions & 6 deletions be/src/vec/exec/format/parquet/schema_desc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ TypeDescriptor FieldDescriptor::get_doris_type(const tparquet::SchemaElement& ph
if (physical_schema.__isset.logicalType) {
type = convert_to_doris_type(physical_schema.logicalType);
} else if (physical_schema.__isset.converted_type) {
type = convert_to_doris_type(physical_schema.converted_type);
type = convert_to_doris_type(physical_schema);
}
// use physical type instead
if (type.type == INVALID_TYPE) {
Expand Down Expand Up @@ -211,7 +211,8 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi
if (logicalType.__isset.STRING) {
type = TypeDescriptor(TYPE_STRING);
} else if (logicalType.__isset.DECIMAL) {
type = TypeDescriptor(TYPE_DECIMALV2);
type = TypeDescriptor::create_decimalv3_type(logicalType.DECIMAL.precision,
logicalType.DECIMAL.scale);
} else if (logicalType.__isset.DATE) {
type = TypeDescriptor(TYPE_DATEV2);
} else if (logicalType.__isset.INTEGER) {
Expand All @@ -238,14 +239,16 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi
return type;
}

TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::type convertedType) {
TypeDescriptor FieldDescriptor::convert_to_doris_type(
const tparquet::SchemaElement& physical_schema) {
TypeDescriptor type;
switch (convertedType) {
switch (physical_schema.converted_type) {
case tparquet::ConvertedType::type::UTF8:
type = TypeDescriptor(TYPE_STRING);
break;
case tparquet::ConvertedType::type::DECIMAL:
type = TypeDescriptor(TYPE_DECIMALV2);
type = TypeDescriptor::create_decimalv3_type(physical_schema.precision,
physical_schema.scale);
break;
case tparquet::ConvertedType::type::DATE:
type = TypeDescriptor(TYPE_DATEV2);
Expand Down Expand Up @@ -281,7 +284,7 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::t
type = TypeDescriptor(TYPE_BIGINT);
break;
default:
LOG(WARNING) << "Not supported parquet ConvertedType: " << convertedType;
LOG(WARNING) << "Not supported parquet ConvertedType: " << physical_schema.converted_type;
type = TypeDescriptor(INVALID_TYPE);
break;
}
Expand Down
2 changes: 1 addition & 1 deletion be/src/vec/exec/format/parquet/schema_desc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class FieldDescriptor {

TypeDescriptor convert_to_doris_type(tparquet::LogicalType logicalType);

TypeDescriptor convert_to_doris_type(tparquet::ConvertedType::type convertedType);
TypeDescriptor convert_to_doris_type(const tparquet::SchemaElement& physical_schema);

TypeDescriptor get_doris_type(const tparquet::SchemaElement& physical_schema);

Expand Down
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,10 @@ public Column(String name, PrimitiveType dataType, boolean isAllowNull) {
this(name, ScalarType.createType(dataType), isAllowNull);
}

public Column(String name, PrimitiveType dataType, int len, int precision, int scale, boolean isAllowNull) {
this(name, ScalarType.createType(dataType, len, precision, scale), isAllowNull);
}

public Column(String name, Type type, boolean isAllowNull) {
this(name, type, false, null, isAllowNull, null, "");
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,7 +797,7 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale) {
if (match.find()) {
scale = Integer.parseInt(match.group(1));
}
return ScalarType.createDecimalType(precision, scale);
return ScalarType.createDecimalV3Type(precision, scale);
}
return Type.UNSUPPORTED;
}
Expand Down
5 changes: 4 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,9 @@ private void fillColumns(InternalService.PFetchTableSchemaResult result)
// only support ScalarType.
PScalarType scalarType = typeNode.getScalarType();
TPrimitiveType tPrimitiveType = TPrimitiveType.findByValue(scalarType.getType());
columns.add(new Column(colName, PrimitiveType.fromThrift(tPrimitiveType), true));
columns.add(new Column(colName, PrimitiveType.fromThrift(tPrimitiveType),
scalarType.getLen() <= 0 ? -1 : scalarType.getLen(), scalarType.getPrecision(),
scalarType.getScale(), true));
}
}
}
Expand Down Expand Up @@ -426,3 +428,4 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws Analysis
.setFileScanRange(ByteString.copyFrom(new TSerializer().serialize(fileScanRange))).build();
}
}

82 changes: 41 additions & 41 deletions regression-test/data/correctness_p0/table_valued_function/test_hdfs_tvf.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,48 +176,48 @@
10 nami 18

-- !parquet --
1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755 each slyly above the careful
2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032 slyly bold instructions. idle dependen
3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192 blithely silent requests after the express dependencies are sl
4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641 riously even requests above the exp
5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283 . slyly regular pinto bea
6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365 final accounts. regular dolphins use against the furiously ironic decoys.
7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627 al pinto beans. asymptotes haggl
9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302 s. unusual, even requests along the furiously regular pac
10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891 ing waters. regular requests ar
11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu
12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432 al packages nag alongside of the bold instructions. express, daring accounts
13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107 requests engage regularly instructions. furiously special requests ar
14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189 l accounts boost. fluffily bold warhorses wake
15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308 across the furiously regular platelets wake even deposits. quickly express she
16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972 ously express ideas haggle quickly dugouts? fu
17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687 eep against the furiously bold ideas. fluffily bold packa
18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040 accounts snooze slyly furiously bold
19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150 refully final foxes across the dogged theodolites sleep slyly abou
20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr
1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755.94 each slyly above the careful
2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032.68 slyly bold instructions. idle dependen
3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192.40 blithely silent requests after the express dependencies are sl
4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641.08 riously even requests above the exp
5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283.84 . slyly regular pinto bea
6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365.79 final accounts. regular dolphins use against the furiously ironic decoys.
7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820.35 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627.85 al pinto beans. asymptotes haggl
9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302.37 s. unusual, even requests along the furiously regular pac
10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891.91 ing waters. regular requests ar
11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393.08 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu
12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432.69 al packages nag alongside of the bold instructions. express, daring accounts
13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107.22 requests engage regularly instructions. furiously special requests ar
14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189.82 l accounts boost. fluffily bold warhorses wake
15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308.56 across the furiously regular platelets wake even deposits. quickly express she
16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972.26 ously express ideas haggle quickly dugouts? fu
17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687.81 eep against the furiously bold ideas. fluffily bold packa
18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040.82 accounts snooze slyly furiously bold
19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150.38 refully final foxes across the dogged theodolites sleep slyly abou
20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530.82 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr

-- !orc --
1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901 ly. slyly ironi
2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902 lar accounts amo
3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903 egular deposits hag
4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904 p furiously r
5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905 wake carefully
6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906 sual a
7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907 lyly. ex
8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908 eposi
9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909 ironic foxe
10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910 ithely final deposit
11 spring maroon seashell almond orchid Manufacturer#2 Brand#25 STANDARD BURNISHED NICKEL 43 WRAP BOX 911 ng gr
12 cornflower wheat orange maroon ghost Manufacturer#3 Brand#33 MEDIUM ANODIZED STEEL 25 JUMBO CASE 912 quickly
13 ghost olive orange rosy thistle Manufacturer#5 Brand#55 MEDIUM BURNISHED NICKEL 1 JUMBO PACK 913 osits.
14 khaki seashell rose cornsilk navajo Manufacturer#1 Brand#13 SMALL POLISHED STEEL 28 JUMBO BOX 914 kages c
15 blanched honeydew sky turquoise medium Manufacturer#1 Brand#15 LARGE ANODIZED BRASS 45 LG CASE 915 usual ac
16 deep sky turquoise drab peach Manufacturer#3 Brand#32 PROMO PLATED TIN 2 MED PACK 916 unts a
17 indian navy coral pink deep Manufacturer#4 Brand#43 ECONOMY BRUSHED STEEL 16 LG BOX 917 regular accounts
18 turquoise indian lemon lavender misty Manufacturer#1 Brand#11 SMALL BURNISHED STEEL 42 JUMBO PACK 918 s cajole slyly a
19 chocolate navy tan deep brown Manufacturer#2 Brand#23 SMALL ANODIZED NICKEL 33 WRAP BOX 919 pending acc
20 ivory navy honeydew sandy midnight Manufacturer#1 Brand#12 LARGE POLISHED NICKEL 48 MED BAG 920 are across the asympt
1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901.00 ly. slyly ironi
2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902.00 lar accounts amo
3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903.00 egular deposits hag
4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904.00 p furiously r
5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905.00 wake carefully
6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906.00 sual a
7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907.00 lyly. ex
8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908.00 eposi
9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909.00 ironic foxe
10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910.01 ithely final deposit
11 spring maroon seashell almond orchid Manufacturer#2 Brand#25 STANDARD BURNISHED NICKEL 43 WRAP BOX 911.01 ng gr
12 cornflower wheat orange maroon ghost Manufacturer#3 Brand#33 MEDIUM ANODIZED STEEL 25 JUMBO CASE 912.01 quickly
13 ghost olive orange rosy thistle Manufacturer#5 Brand#55 MEDIUM BURNISHED NICKEL 1 JUMBO PACK 913.01 osits.
14 khaki seashell rose cornsilk navajo Manufacturer#1 Brand#13 SMALL POLISHED STEEL 28 JUMBO BOX 914.01 kages c
15 blanched honeydew sky turquoise medium Manufacturer#1 Brand#15 LARGE ANODIZED BRASS 45 LG CASE 915.01 usual ac
16 deep sky turquoise drab peach Manufacturer#3 Brand#32 PROMO PLATED TIN 2 MED PACK 916.01 unts a
17 indian navy coral pink deep Manufacturer#4 Brand#43 ECONOMY BRUSHED STEEL 16 LG BOX 917.01 regular accounts
18 turquoise indian lemon lavender misty Manufacturer#1 Brand#11 SMALL BURNISHED STEEL 42 JUMBO PACK 918.01 s cajole slyly a
19 chocolate navy tan deep brown Manufacturer#2 Brand#23 SMALL ANODIZED NICKEL 33 WRAP BOX 919.01 pending acc
20 ivory navy honeydew sandy midnight Manufacturer#1 Brand#12 LARGE POLISHED NICKEL 48 MED BAG 920.02 are across the asympt

-- !json --
1 beijing 2345671
Expand Down Expand Up @@ -293,6 +293,6 @@ s_name TEXT Yes false \N NONE
s_address TEXT Yes false \N NONE
s_nationkey INT Yes false \N NONE
s_phone TEXT Yes false \N NONE
s_acctbal DECIMAL(9, 0) Yes false \N NONE
s_acctbal DECIMAL(12, 2) Yes false \N NONE
s_comment TEXT Yes false \N NONE

Loading

0 comments on commit 2929099

Please sign in to comment.