Skip to content

Commit

Permalink
improve shape case
Browse files Browse the repository at this point in the history
  • Loading branch information
@englefly
englefly committed Dec 8, 2023
1 parent ddb6eb5 commit 8f5c2ca
Show file tree
Hide file tree
Showing 323 changed files with 9,083 additions and 8,415 deletions.
6 changes: 5 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,21 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
static final double BROADCAST_JOIN_SKEW_PENALTY_LIMIT = 2.0;
static final double RANDOM_SHUFFLE_TO_HASH_SHUFFLE_FACTOR = 0.1;
private final int beNumber;
private final int parallelInstance;

public CostModelV1(ConnectContext connectContext) {
SessionVariable sessionVariable = connectContext.getSessionVariable();
if (sessionVariable.isPlayNereidsDump()) {
// TODO: @bingfeng refine minidump setting, and pass testMinidumpUt
beNumber = 1;
parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum());
} else if (sessionVariable.getBeNumberForTest() != -1) {
// shape test, fix the BE number and instance number
beNumber = sessionVariable.getBeNumberForTest();
parallelInstance = 8;
} else {
beNumber = Math.max(1, ConnectContext.get().getEnv().getClusterInfo().getBackendsNumber(true));
parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum());
}
}

Expand Down Expand Up @@ -287,7 +292,6 @@ public Cost visitPhysicalHashJoin(
// on the output rows, taken on outputRowCount()
double probeSideFactor = 1.0;
double buildSideFactor = context.getSessionVariable().getBroadcastRightTableScaleFactor();
int parallelInstance = Math.max(1, context.getSessionVariable().getParallelExecInstanceNum());
int totalInstanceNumber = parallelInstance * beNumber;
if (buildSideFactor <= 1.0) {
if (buildStats.computeSize() < 1024 * 1024) {
Expand Down
480 changes: 480 additions & 0 deletions regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/checkTpcds1000ColStats.out
View file

Large diffs are not rendered by default.

159 changes: 0 additions & 159 deletions regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query14.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,162 +175,3 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
--------------------------------PhysicalDistribute
----------------------------------PhysicalCteConsumer ( cteId=CTEId#1 )

-- !ds_shape_14_2 --
PhysicalCteAnchor ( cteId=CTEId#0 )
--PhysicalCteProducer ( cteId=CTEId#0 )
----PhysicalProject
------hashJoin[INNER_JOIN] hashCondition=((item.i_brand_id = x.brand_id) and (item.i_category_id = x.category_id) and (item.i_class_id = x.class_id)) otherCondition=() build RFs:RF6 class_id->[i_class_id];RF7 category_id->[i_category_id];RF8 brand_id->[i_brand_id]
--------PhysicalProject
----------PhysicalOlapScan[item] apply RFs: RF6 RF7 RF8
--------PhysicalDistribute
----------PhysicalIntersect
------------hashAgg[GLOBAL]
--------------PhysicalDistribute
----------------hashAgg[LOCAL]
------------------PhysicalProject
--------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = iss.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[ss_item_sk]
----------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
------------------------PhysicalProject
--------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------filter((d1.d_year <= 2001) and (d1.d_year >= 1999))
------------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalDistribute
------------------------PhysicalProject
--------------------------PhysicalOlapScan[item]
------------hashAgg[GLOBAL]
--------------PhysicalDistribute
----------------hashAgg[LOCAL]
------------------PhysicalProject
--------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = ics.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[cs_item_sk]
----------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
------------------------PhysicalProject
--------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------filter((d2.d_year <= 2001) and (d2.d_year >= 1999))
------------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalDistribute
------------------------PhysicalProject
--------------------------PhysicalOlapScan[item]
------------hashAgg[GLOBAL]
--------------PhysicalDistribute
----------------hashAgg[LOCAL]
------------------PhysicalProject
--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = iws.i_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[ws_item_sk]
----------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = d3.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_sold_date_sk]
------------------------PhysicalProject
--------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------filter((d3.d_year <= 2001) and (d3.d_year >= 1999))
------------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalDistribute
------------------------PhysicalProject
--------------------------PhysicalOlapScan[item]
--PhysicalCteAnchor ( cteId=CTEId#1 )
----PhysicalCteProducer ( cteId=CTEId#1 )
------hashAgg[GLOBAL]
--------PhysicalDistribute
----------hashAgg[LOCAL]
------------PhysicalUnion
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[ss_sold_date_sk]
--------------------PhysicalProject
----------------------PhysicalOlapScan[store_sales] apply RFs: RF9
--------------------PhysicalDistribute
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[cs_sold_date_sk]
--------------------PhysicalProject
----------------------PhysicalOlapScan[catalog_sales] apply RFs: RF10
--------------------PhysicalDistribute
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF11 d_date_sk->[ws_sold_date_sk]
--------------------PhysicalProject
----------------------PhysicalOlapScan[web_sales] apply RFs: RF11
--------------------PhysicalDistribute
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
----PhysicalResultSink
------PhysicalTopN[MERGE_SORT]
--------PhysicalDistribute
----------PhysicalTopN[LOCAL_SORT]
------------hashJoin[INNER_JOIN] hashCondition=((ty_brand = ly_brand) and (ty_category = ly_category) and (ty_class = ly_class)) otherCondition=()
--------------PhysicalProject
----------------NestedLoopJoin[INNER_JOIN](cast(ty_sales as DOUBLE) > cast(average_sales as DOUBLE))
------------------PhysicalProject
--------------------hashAgg[GLOBAL]
----------------------PhysicalDistribute
------------------------hashAgg[LOCAL]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 ss_item_sk->[i_item_sk]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[item] apply RFs: RF19
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF18 ss_item_sk->[ss_item_sk]
------------------------------------PhysicalDistribute
--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
------------------------------------PhysicalDistribute
--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF17 d_date_sk->[ss_sold_date_sk]
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF17
----------------------------------------PhysicalDistribute
------------------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_week_seq = date_dim.d_week_seq)) otherCondition=() build RFs:RF16 d_week_seq->[d_week_seq]
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF16
--------------------------------------------PhysicalDistribute
----------------------------------------------PhysicalAssertNumRows
------------------------------------------------PhysicalDistribute
--------------------------------------------------PhysicalProject
----------------------------------------------------filter((date_dim.d_dom = 3) and (date_dim.d_moy = 12) and (date_dim.d_year = 2000))
------------------------------------------------------PhysicalOlapScan[date_dim]
------------------PhysicalDistribute
--------------------PhysicalAssertNumRows
----------------------PhysicalDistribute
------------------------PhysicalCteConsumer ( cteId=CTEId#1 )
--------------PhysicalProject
----------------NestedLoopJoin[INNER_JOIN](cast(ly_sales as DOUBLE) > cast(average_sales as DOUBLE))
------------------PhysicalProject
--------------------hashAgg[GLOBAL]
----------------------PhysicalDistribute
------------------------hashAgg[LOCAL]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 ss_item_sk->[i_item_sk]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[item] apply RFs: RF15
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[ss_item_sk]
------------------------------------PhysicalDistribute
--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
------------------------------------PhysicalDistribute
--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[ss_sold_date_sk]
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF13
----------------------------------------PhysicalDistribute
------------------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_week_seq = date_dim.d_week_seq)) otherCondition=() build RFs:RF12 d_week_seq->[d_week_seq]
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF12
--------------------------------------------PhysicalDistribute
----------------------------------------------PhysicalAssertNumRows
------------------------------------------------PhysicalDistribute
--------------------------------------------------PhysicalProject
----------------------------------------------------filter((date_dim.d_dom = 3) and (date_dim.d_moy = 12) and (date_dim.d_year = 1999))
------------------------------------------------------PhysicalOlapScan[date_dim]
------------------PhysicalDistribute
--------------------PhysicalAssertNumRows
----------------------PhysicalDistribute
------------------------PhysicalCteConsumer ( cteId=CTEId#1 )

118 changes: 0 additions & 118 deletions regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,121 +99,3 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------------------PhysicalProject
--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )

-- !ds_shape_23_2 --
PhysicalCteAnchor ( cteId=CTEId#0 )
--PhysicalCteProducer ( cteId=CTEId#0 )
----PhysicalProject
------filter((cnt > 4))
--------hashAgg[GLOBAL]
----------PhysicalDistribute
------------hashAgg[LOCAL]
--------------PhysicalProject
----------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[ss_item_sk]
------------------PhysicalProject
--------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
----------------------PhysicalProject
------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
----------------------PhysicalDistribute
------------------------PhysicalProject
--------------------------filter(d_year IN (2000, 2001, 2002, 2003))
----------------------------PhysicalOlapScan[date_dim]
------------------PhysicalDistribute
--------------------PhysicalProject
----------------------PhysicalOlapScan[item]
--PhysicalCteAnchor ( cteId=CTEId#2 )
----PhysicalCteProducer ( cteId=CTEId#2 )
------PhysicalProject
--------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast((0.9500 * tpcds_cmax) as DOUBLE))
----------hashAgg[LOCAL]
------------PhysicalProject
--------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
----------------PhysicalDistribute
------------------PhysicalProject
--------------------PhysicalOlapScan[store_sales] apply RFs: RF4
----------------PhysicalDistribute
------------------PhysicalProject
--------------------PhysicalOlapScan[customer]
----------PhysicalDistribute
------------PhysicalProject
--------------hashAgg[GLOBAL]
----------------PhysicalDistribute
------------------hashAgg[LOCAL]
--------------------PhysicalProject
----------------------hashAgg[LOCAL]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
----------------------------PhysicalDistribute
------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
--------------------------------PhysicalProject
----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
--------------------------------------PhysicalOlapScan[date_dim]
----------------------------PhysicalDistribute
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[customer]
----PhysicalResultSink
------PhysicalTopN[MERGE_SORT]
--------PhysicalDistribute
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalUnion
--------------PhysicalDistribute
----------------PhysicalTopN[MERGE_SORT]
------------------PhysicalDistribute
--------------------PhysicalTopN[LOCAL_SORT]
----------------------hashAgg[GLOBAL]
------------------------PhysicalDistribute
--------------------------hashAgg[LOCAL]
----------------------------PhysicalProject
------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------------PhysicalDistribute
----------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[cs_bill_customer_sk]
------------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[cs_bill_customer_sk]
--------------------------------------PhysicalDistribute
----------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7
------------------------------------------PhysicalDistribute
--------------------------------------------PhysicalProject
----------------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
------------------------------------------------PhysicalOlapScan[date_dim]
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------PhysicalOlapScan[customer]
--------------PhysicalDistribute
----------------PhysicalTopN[MERGE_SORT]
------------------PhysicalDistribute
--------------------PhysicalTopN[LOCAL_SORT]
----------------------hashAgg[GLOBAL]
------------------------PhysicalDistribute
--------------------------hashAgg[LOCAL]
----------------------------PhysicalProject
------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=()
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------------PhysicalDistribute
----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF10 ws_bill_customer_sk->[c_customer_sk]
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------PhysicalOlapScan[customer] apply RFs: RF10
------------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF9 c_customer_sk->[ws_bill_customer_sk]
--------------------------------------PhysicalDistribute
----------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9
------------------------------------------PhysicalDistribute
--------------------------------------------PhysicalProject
----------------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
------------------------------------------------PhysicalOlapScan[date_dim]
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )

Loading

0 comments on commit 8f5c2ca

Please sign in to comment.