From 841eea790dac371894947a2c7a6b70643e63a883 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 19 Jul 2024 14:38:12 +0800 Subject: [PATCH 1/6] 1 --- .../java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java index a42fae22585873..d7bb1080c5929e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java @@ -78,6 +78,8 @@ public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException "partitionColumnType should be date/datetime " + "when PartitionType is range and expr is date_trunc"); } + } else { + throw new AnalysisException("date_trunc only support range partition"); } } From 96e398f7cf06abf8bf2132e8616f569c1c6f6c1a Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 19 Jul 2024 14:52:35 +0800 Subject: [PATCH 2/6] 1 --- .../test_hive_limit_partition_mtmv.out | 5 - .../test_hive_limit_partition_mtmv.groovy | 45 +-- .../mtmv_p0/test_rollup_partition_mtmv.groovy | 334 +----------------- 3 files changed, 31 insertions(+), 353 deletions(-) diff --git a/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out b/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out index 1a8731d6a1c446..48e60bb46cc027 100644 --- a/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out +++ b/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out @@ -11,11 +11,6 @@ 2 20380101 sh 5 20380102 bj --- !mtmv_datetrunc -- -1 20380101 bj -2 20380101 sh -5 20380102 bj - -- !select_base_table -- 1 bj 20380101 2 sh 20380101 diff --git a/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy index 1e6c49bb50af0e..ea5c2ff2844591 100644 --- a/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy @@ -115,34 +115,23 @@ suite("test_hive_limit_partition_mtmv", "p0,external,hive,external_docker,extern // date trunc sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by (date_trunc(`day`,'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1', - 'partition_sync_limit'='2', - 'partition_sync_time_unit'='MONTH', - 'partition_date_format'='%Y%m%d' - ) - AS - SELECT k1,day,region FROM ${catalog_name}.${hive_database}.${hive_table}; - """ - showPartitionsResult = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertEquals(1, showPartitionsResult.size()) - assertTrue(showPartitionsResult.toString().contains("_20380101")) - assertTrue(showPartitionsResult.toString().contains("_20380102")) - - // refresh complete - sql """ - REFRESH MATERIALIZED VIEW ${mvName} complete - """ - jobName = getJobName(dbName, mvName); - waitingMTMVTaskFinished(jobName) - order_qt_mtmv_datetrunc "SELECT * FROM ${mvName} order by k1,day,region" - + test { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`day`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='MONTH', + 'partition_date_format'='%Y%m%d' + ) + AS + SELECT k1,day,region FROM ${catalog_name}.${hive_database}.${hive_table}; + """ + exception "only support" + } // date type sql """drop materialized view if exists ${mvName};""" diff --git a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy index c24bc5aa91085a..3b8f943f7ba4fa 100644 --- a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy @@ -46,326 +46,20 @@ suite("test_rollup_partition_mtmv") { insert into ${tableName} values(1,"2020-01-01", "2020-01-01"),(2,"2020-01-02", "2020-01-02"),(3,"2020-02-01", "2020-02-01"); """ - // list date month - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by (date_trunc(`k2`,'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT * FROM ${tableName}; - """ - showPartitionsResult = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertEquals(2, showPartitionsResult.size()) - - sql """ - REFRESH MATERIALIZED VIEW ${mvName} AUTO - """ - def jobName = getJobName(dbName, mvName); - log.info(jobName) - waitingMTMVTaskFinished(jobName) - order_qt_date_list_month "SELECT * FROM ${mvName} order by k1,k2" - - sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (month_alias) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; - """ - def date_list_month_partitions = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + date_list_month_partitions.toString()) - assertEquals(2, date_list_month_partitions.size()) - waitingMTMVTaskFinished(getJobName(dbName, mvName)) - order_qt_date_list_month_partition_by_column "SELECT * FROM ${mvName}" - - sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName}; - """ - def date_list_month_partitions_level = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + date_list_month_partitions_level.toString()) - assertEquals(2, date_list_month_partitions_level.size()) - waitingMTMVTaskFinished(getJobName(dbName, mvName)) - order_qt_date_list_month_level "SELECT * FROM ${mvName}" - - - sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias, k1, count(*) FROM ${tableName} group by month_alias, k1; - """ - def date_list_month_partitions_level_agg = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + date_list_month_partitions_level_agg.toString()) - assertEquals(2, date_list_month_partitions_level_agg.size()) - waitingMTMVTaskFinished(getJobName(dbName, mvName)) - order_qt_date_list_month_level_agg "SELECT * FROM ${mvName}" - - - sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias, k3, count(*) FROM ${tableName} group by date_trunc(`k2`,'day'), k3; - """ - def date_list_month_partitions_level_agg_multi = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + date_list_month_partitions_level_agg_multi.toString()) - assertEquals(2, date_list_month_partitions_level_agg_multi.size()) - waitingMTMVTaskFinished(getJobName(dbName, mvName)) - order_qt_date_list_month_level_agg_multi "SELECT * FROM ${mvName}" - - - sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias, count(*) FROM ${tableName} group by k2; - """ - def date_list_month_partitions_level_agg_direct = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + date_list_month_partitions_level_agg_direct.toString()) - assertEquals(2, date_list_month_partitions_level_agg_direct.size()) - waitingMTMVTaskFinished(getJobName(dbName, mvName)) - order_qt_date_list_month_level_agg "SELECT * FROM ${mvName}" - - - - // mv partition level should be higher or equal then query, should fail - sql """drop materialized view if exists ${mvName};""" - try { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'day')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("partition column time unit level should be greater than sql select column")) - } - - // mv partition use a column not in mv sql select, should fail - sql """drop materialized view if exists ${mvName};""" - try { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(`k2`, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("partition column can not find from sql select column")) - } - - sql """drop materialized view if exists ${mvName};""" - // list date year - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by (date_trunc(`k2`,'year')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT * FROM ${tableName}; - """ - showPartitionsResult = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertEquals(1, showPartitionsResult.size()) - - sql """drop materialized view if exists ${mvName};""" - // list date year - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (year_alias) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'year') as year_alias, * FROM ${tableName}; - """ - def date_list_year_partitions = sql """show partitions from ${mvName}""" - assertEquals(1, date_list_year_partitions.size()) - order_qt_date_list_year_partition_by_column "SELECT * FROM ${mvName}" - - // list string month - sql """drop table if exists `${tableName}`""" - sql """drop materialized view if exists ${mvName};""" - sql """ - CREATE TABLE `${tableName}` ( - `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', - `k2` varchar(200) NOT NULL COMMENT '\"数据灌入日期时间\"' - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - PARTITION BY list(`k2`) - ( - PARTITION p_20200101 VALUES IN ("2020==01==01"), - PARTITION p_20200102 VALUES IN ("2020==01==02"), - PARTITION p_20200201 VALUES IN ("2020==02==01") - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 2 - PROPERTIES ('replication_num' = '1') ; - """ - sql """ - insert into ${tableName} values(1,"2020==01==01"),(2,"2020==01==02"),(3,"2020==02==01"); - """ - - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by (date_trunc(`k2`,'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1', - 'partition_date_format'='%Y==%m==%d' - ) - AS - SELECT * FROM ${tableName}; - """ - showPartitionsResult = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertEquals(2, showPartitionsResult.size()) - - sql """ - REFRESH MATERIALIZED VIEW ${mvName} AUTO - """ - jobName = getJobName(dbName, mvName); - log.info(jobName) - waitingMTMVTaskFinished(jobName) - order_qt_string_list_month "SELECT * FROM ${mvName} order by k1,k2" - - - sql """drop materialized view if exists ${mvName};""" - try { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("use invalid implicit expression")) - } - - // mv partition level should be higher or equal then query, should fail - sql """drop materialized view if exists ${mvName};""" - try { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(month_alias, 'day')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("use invalid implicit expression")) - } - - // mv partition use a column not in mv sql select, should fail - sql """drop materialized view if exists ${mvName};""" - try { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (date_trunc(`k2`, 'month')) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("partition column can not find from sql select column")) - } - - // mv partition column type is date, base table is string, partition mapping fail - // support later - sql """drop materialized view if exists ${mvName};""" - try { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH AUTO ON MANUAL - partition by (month_alias) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1', - 'partition_date_format'='%Y==%m==%d' - ) - AS - SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - } + test { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + exception "only support" + } // range date month sql """drop table if exists `${tableName}`""" From 0ddf9c7fcea657a9ec7ae6bd842e4cd312f1afa4 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 19 Jul 2024 14:55:41 +0800 Subject: [PATCH 3/6] 1 --- .../mtmv_p0/test_rollup_partition_mtmv.out | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out index 5f01fb0ee29ad9..f828e65cc70b39 100644 --- a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out +++ b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out @@ -1,41 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !date_list_month -- -1 2020-01-01 2020-01-01 -2 2020-01-02 2020-01-02 -3 2020-02-01 2020-02-01 - --- !date_list_month_partition_by_column -- -2020-01-01 1 2020-01-01 2020-01-01 -2020-01-01 2 2020-01-02 2020-01-02 -2020-02-01 3 2020-02-01 2020-02-01 - --- !date_list_month_level -- -2020-01-01 1 2020-01-01 2020-01-01 -2020-01-02 2 2020-01-02 2020-01-02 -2020-02-01 3 2020-02-01 2020-02-01 - --- !date_list_month_level_agg -- -2020-01-01 1 1 -2020-01-02 2 1 -2020-02-01 3 1 - --- !date_list_month_level_agg_multi -- -2020-01-01 2020-01-01 1 -2020-01-02 2020-01-02 1 -2020-02-01 2020-02-01 1 - --- !date_list_month_level_agg -- -2020-01-01 1 -2020-01-02 1 -2020-02-01 1 - --- !date_list_year_partition_by_column -- - --- !string_list_month -- -1 2020==01==01 -2 2020==01==02 -3 2020==02==01 - -- !date_range_month -- 1 2020-01-01 2020-01-01 2 2020-01-02 2020-01-02 From 3de1567925c556e169eff21fdea7d3e597022661 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 19 Jul 2024 21:22:48 +0800 Subject: [PATCH 4/6] 1 --- regression-test/suites/mtmv_p0/test_build_mtmv.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy index 580b9f9bbfc2b7..46d0b26cc5f4be 100644 --- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy @@ -501,10 +501,10 @@ suite("test_build_mtmv") { DROP MATERIALIZED VIEW ${mvName} """ def jobs = sql """select count(1) from jobs("type"="mv") where name= '${jobName}'""" - println jobs + log.info(jobs) assertEquals(jobs.get(0).get(0), 0); def tasks = sql """select count(1) from tasks("type"="mv") where jobname = '${jobName}'""" - println tasks + log.info(tasks) assertEquals(tasks.get(0).get(0), 0); // test bitmap From 1b523e14770fa184c7f23cdb284644e78d4324f2 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 22 Jul 2024 10:41:01 +0800 Subject: [PATCH 5/6] 1 --- regression-test/suites/mtmv_p0/test_build_mtmv.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy index 46d0b26cc5f4be..0b24a1b553c659 100644 --- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy @@ -501,10 +501,10 @@ suite("test_build_mtmv") { DROP MATERIALIZED VIEW ${mvName} """ def jobs = sql """select count(1) from jobs("type"="mv") where name= '${jobName}'""" - log.info(jobs) + log.info(jobs.toString()) assertEquals(jobs.get(0).get(0), 0); def tasks = sql """select count(1) from tasks("type"="mv") where jobname = '${jobName}'""" - log.info(tasks) + log.info(tasks.toString()) assertEquals(tasks.get(0).get(0), 0); // test bitmap From 66c767ea4892cbda0db303fd6724b3366c58f71a Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 22 Jul 2024 16:30:24 +0800 Subject: [PATCH 6/6] 1 --- regression-test/suites/mtmv_p0/test_build_mtmv.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy index 0b24a1b553c659..ae1af18fdc5816 100644 --- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy @@ -500,12 +500,12 @@ suite("test_build_mtmv") { sql """ DROP MATERIALIZED VIEW ${mvName} """ - def jobs = sql """select count(1) from jobs("type"="mv") where name= '${jobName}'""" + def jobs = sql """select * from jobs("type"="mv") where MvName= '${mvName}'""" log.info(jobs.toString()) - assertEquals(jobs.get(0).get(0), 0); - def tasks = sql """select count(1) from tasks("type"="mv") where jobname = '${jobName}'""" + assertEquals(0, jobs.size()); + def tasks = sql """select * from tasks("type"="mv") where MvName = '${mvName}'""" log.info(tasks.toString()) - assertEquals(tasks.get(0).get(0), 0); + assertEquals(0, tasks.size()); // test bitmap sql """drop table if exists `${tableName}`"""