From e16998878fd713baddc7d7e96c60ebf338853736 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Wed, 9 Feb 2022 18:16:56 -0800 Subject: [PATCH 01/24] Fix back-patch of "Avoid race in RelationBuildDesc() ..." The back-patch of commit fdd965d074d46765c295223b119ca437dbcac973 broke CLOBBER_CACHE_ALWAYS for v9.6 through v13. It updated the InvalidateSystemCaches() call for CLOBBER_CACHE_RECURSIVELY, neglecting the one for CLOBBER_CACHE_ALWAYS. Back-patch to v13, v12, v11, and v10. Reviewed by Tomas Vondra. Reported by Tomas Vondra. Discussion: https://postgr.es/m/df7b4c0b-7d92-f03f-75c4-9e08b269a716@enterprisedb.com --- src/backend/utils/cache/inval.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/cache/inval.c b/src/backend/utils/cache/inval.c index 5c41466a4ff..5a986349e2a 100644 --- a/src/backend/utils/cache/inval.c +++ b/src/backend/utils/cache/inval.c @@ -785,7 +785,18 @@ AcceptInvalidationMessages(void) * recursive reloads it's unlikely you'll learn more. *---------- */ -#ifdef DISCARD_CACHES_ENABLED +#if defined(CLOBBER_CACHE_ALWAYS) + { + static bool in_recursion = false; + + if (!in_recursion) + { + in_recursion = true; + InvalidateSystemCachesExtended(true); + in_recursion = false; + } + } +#elif defined(CLOBBER_CACHE_RECURSIVELY) { static int recursion_depth = 0; From fba4a2d5c85196f3e512297b7190f596b8746350 Mon Sep 17 00:00:00 2001 From: Zhenghua Lyu Date: Thu, 10 Feb 2022 13:24:03 +0800 Subject: [PATCH 02/24] Fix getResUsage integer overflow. In the function getResUsage it will set the group Id (type Oid)'s value from a string. Previous code use pg_atoi to do the value parse, this is not correct because type Oid is uint, we should use atooid. Another place is building the SQL involving group id it uses "%d", this commits fix this by using "%u". --- src/backend/access/transam/varsup.c | 21 +++++++++++++ src/backend/utils/resgroup/resgroup_helper.c | 5 ++- .../resgroup/resgroup_large_group_id.out | 31 +++++++++++++++++++ .../isolation2/isolation2_resgroup_schedule | 3 ++ .../sql/resgroup/resgroup_large_group_id.sql | 13 ++++++++ 5 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 src/test/isolation2/expected/resgroup/resgroup_large_group_id.out create mode 100644 src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql diff --git a/src/backend/access/transam/varsup.c b/src/backend/access/transam/varsup.c index fc866ba9d97..5ca18d1b77d 100644 --- a/src/backend/access/transam/varsup.c +++ b/src/backend/access/transam/varsup.c @@ -26,6 +26,7 @@ #include "storage/pmsignal.h" #include "storage/proc.h" #include "cdb/cdbutil.h" +#include "utils/faultinjector.h" #include "utils/guc.h" #include "utils/syscache.h" @@ -624,6 +625,26 @@ GetNewObjectIdUnderLock(void) (ShmemVariableCache->nextOid)++; (ShmemVariableCache->oidCount)--; +#ifdef FAULT_INJECTOR + if (SIMPLE_FAULT_INJECTOR("bump_oid") == FaultInjectorTypeSkip) + { + /* + * CDB: we encounter high oid issues several times, we should + * have some test-utils to verify logic under larger oid. + * + * NOTE: we do not have undo-bump, so take care when you decide to + * use this fault inject. Currently, only resgroup test job uses it, + * that is safe, becase resgroup job is an independent pipeline job. + */ + Oid large_oid = (1U<<31)+5; /* this value will overflow if taken as int32 */ + if (ShmemVariableCache->nextOid < large_oid) + { + ShmemVariableCache->nextOid = large_oid + 1; + result = large_oid; + } + } +#endif + return result; } diff --git a/src/backend/utils/resgroup/resgroup_helper.c b/src/backend/utils/resgroup/resgroup_helper.c index 68941a639c8..2148ffe3e34 100644 --- a/src/backend/utils/resgroup/resgroup_helper.c +++ b/src/backend/utils/resgroup/resgroup_helper.c @@ -105,7 +105,7 @@ getResUsage(ResGroupStatCtx *ctx, Oid inGroupId) initStringInfo(&buffer); appendStringInfo(&buffer, "SELECT groupid, cpu_usage, memory_usage " - "FROM pg_resgroup_get_status(%d)", + "FROM pg_resgroup_get_status(%u)", inGroupId); CdbDispatchCommand(buffer.data, DF_WITH_SNAPSHOT, &cdb_pgresults); @@ -133,8 +133,7 @@ getResUsage(ResGroupStatCtx *ctx, Oid inGroupId) { const char *result; ResGroupStat *row = &ctx->groups[j]; - Oid groupId = pg_atoi(PQgetvalue(pg_result, j, 0), - sizeof(Oid), 0); + Oid groupId = atooid(PQgetvalue(pg_result, j, 0)); Assert(groupId == row->groupId); diff --git a/src/test/isolation2/expected/resgroup/resgroup_large_group_id.out b/src/test/isolation2/expected/resgroup/resgroup_large_group_id.out new file mode 100644 index 00000000000..558cfad58a3 --- /dev/null +++ b/src/test/isolation2/expected/resgroup/resgroup_large_group_id.out @@ -0,0 +1,31 @@ +-- Test resgroup oid larger than int32. +select gp_inject_fault('bump_oid', 'skip', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) + +create resource group rg_large_oid with (cpu_rate_limit=20, memory_limit=10); +CREATE + +select gp_inject_fault('bump_oid', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) + +select max(oid)::bigint > (power(2,31) + 1)::bigint from pg_resgroup; + ?column? +---------- + t +(1 row) + +-- count(*) > 0 to run the SQL but do not display the result +select count(*) > 0 from pg_resgroup_get_status(NULL); + ?column? +---------- + t +(1 row) + +drop resource group rg_large_oid; +DROP diff --git a/src/test/isolation2/isolation2_resgroup_schedule b/src/test/isolation2/isolation2_resgroup_schedule index e94ee880a4b..c6aad426821 100644 --- a/src/test/isolation2/isolation2_resgroup_schedule +++ b/src/test/isolation2/isolation2_resgroup_schedule @@ -51,4 +51,7 @@ test: resgroup/resgroup_functions # dump info test: resgroup/resgroup_dumpinfo +# test larget group id +test: resgroup/resgroup_large_group_id + test: resgroup/disable_resgroup diff --git a/src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql b/src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql new file mode 100644 index 00000000000..10ec72fa8b6 --- /dev/null +++ b/src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql @@ -0,0 +1,13 @@ +-- Test resgroup oid larger than int32. +select gp_inject_fault('bump_oid', 'skip', dbid) from gp_segment_configuration where role = 'p' and content = -1; + +create resource group rg_large_oid with (cpu_rate_limit=20, memory_limit=10); + +select gp_inject_fault('bump_oid', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1; + +select max(oid)::bigint > (power(2,31) + 1)::bigint from pg_resgroup; + +-- count(*) > 0 to run the SQL but do not display the result +select count(*) > 0 from pg_resgroup_get_status(NULL); + +drop resource group rg_large_oid; From 1dee14dbf135052fb6b176591486558c0437d188 Mon Sep 17 00:00:00 2001 From: FairyFar Date: Fri, 11 Feb 2022 23:23:04 +0800 Subject: [PATCH 03/24] removed redefined PG_AUTOCONF_FILENAME in guc.h (#13081) --- src/include/utils/guc.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index d49f00a5bab..9d7f5f8909e 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -29,14 +29,6 @@ */ #define RESERVED_FTS_CONNECTIONS (1) - -/* - * Automatic configuration file name for ALTER SYSTEM. - * This file will be used to store values of configuration parameters - * set by ALTER SYSTEM command. - */ -#define PG_AUTOCONF_FILENAME "postgresql.auto.conf" - /* upper limit for GUC variables measured in kilobytes of memory */ /* note that various places assume the byte size fits in a "long" variable */ #if SIZEOF_SIZE_T > 4 && SIZEOF_LONG > 4 From 1f139d3ff4e7f837b9df52e40c3c52c0f3d8375d Mon Sep 17 00:00:00 2001 From: David Kimura Date: Mon, 31 Jan 2022 17:55:03 +0000 Subject: [PATCH 04/24] Pass subquery context through callstack ESubqueryCtxt indicates whether a subquery appears in the project list (EsqctxtValue) or comparison predicate (EsqctxtFilter). Currently we can incorrectly recalculate the context inside PexprSubqueryPred(). Following query incorrectly determined the subquery was in the comparison predicate: ``` SELECT (SELECT 1)=ALL(SELECT generate_series(1,2)); ``` Using the incorrect subquery context can cause ORCA to create a plan that incorrect creates an inner (instead of left outer) correlated apply operator. Ultimately, that mistake can lead to wrong results. Fix is the pass the subquery context when available rather than recalculate. --- .../libgpopt/include/gpopt/xforms/CSubqueryHandler.h | 3 ++- .../gporca/libgpopt/src/xforms/CSubqueryHandler.cpp | 12 ++++++------ .../gporca/libgpopt/src/xforms/CXformUtils.cpp | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h b/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h index bfe1d0987d7..b4dd303abf1 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h +++ b/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h @@ -272,7 +272,8 @@ class CSubqueryHandler // build an expression for the quantified comparison of the subquery CExpression *PexprSubqueryPred(CExpression *pexprOuter, CExpression *pexprSubquery, - CExpression **ppexprResult); + CExpression **ppexprResult, + CSubqueryHandler::ESubqueryCtxt esqctxt); // main driver BOOL FProcess( diff --git a/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp b/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp index 5ab73c4ffd8..970de9e728d 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp @@ -179,7 +179,8 @@ CSubqueryHandler::PexprReplace(CMemoryPool *mp, CExpression *pexprInput, CExpression * CSubqueryHandler::PexprSubqueryPred(CExpression *pexprOuter, CExpression *pexprSubquery, - CExpression **ppexprResult) + CExpression **ppexprResult, + CSubqueryHandler::ESubqueryCtxt esqctxt) { GPOS_ASSERT(CUtils::FQuantifiedSubquery(pexprSubquery->Pop())); @@ -187,7 +188,6 @@ CSubqueryHandler::PexprSubqueryPred(CExpression *pexprOuter, CExpression *pexprNewLogical = nullptr; CExpression *pexprScalarChild = (*pexprSubquery)[1]; - CSubqueryHandler::ESubqueryCtxt esqctxt = CSubqueryHandler::EsqctxtFilter; // If pexprScalarChild is a non-scalar subquery such as follows, // EXPLAIN SELECT * FROM t3 WHERE (c = ANY(SELECT c FROM t2)) IN (SELECT b from t1); @@ -1227,7 +1227,7 @@ CSubqueryHandler::FCreateCorrelatedApplyForQuantifiedSubquery( CExpression *pexprResult = nullptr; CSubqueryHandler sh(mp, true /* fEnforceCorrelatedApply */); CExpression *pexprPredicate = - sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult, esqctxt); pexprInner->AddRef(); if (EsqctxtFilter == esqctxt) @@ -1452,7 +1452,7 @@ CSubqueryHandler::FRemoveAnySubquery(CExpression *pexprOuter, // build subquery quantified comparison CExpression *pexprResult = nullptr; CExpression *pexprPredicate = - PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult, esqctxt); // generate a select for the quantified predicate pexprInner->AddRef(); @@ -1623,8 +1623,8 @@ CSubqueryHandler::FRemoveAllSubquery(CExpression *pexprOuter, { // build subquery quantified comparison CExpression *pexprResult = nullptr; - CExpression *pexprPredicate = - PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + CExpression *pexprPredicate = PexprSubqueryPred( + pexprInner, pexprSubquery, &pexprResult, esqctxt); *ppexprResidualScalar = CUtils::PexprScalarConstBool(mp, true /*value*/); diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp index 9472b5b9524..d206be48c80 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp @@ -847,7 +847,8 @@ CXformUtils::SubqueryAnyToAgg( CExpression *pexprResult = nullptr; CSubqueryHandler sh(mp, false /* fEnforceCorrelatedApply */); CExpression *pexprSubqPred = - sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult, + CSubqueryHandler::EsqctxtFilter); CScalarCmp *scalarCmp = CScalarCmp::PopConvert(pexprSubqPred->Pop()); GPOS_ASSERT(nullptr != scalarCmp); From 7da8a5d22ea598d8072f20096038c989e1eaf8ce Mon Sep 17 00:00:00 2001 From: David Kimura Date: Mon, 31 Jan 2022 18:51:31 +0000 Subject: [PATCH 05/24] Restrict predicate push down on left outer correlated apply Given the following query: ``` SELECT 3 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); ``` Without predicate push down: ``` Physical plan: +--CPhysicalComputeScalar |--CPhysicalCorrelatedLeftOuterNLJoin | |--CPhysicalConstTableGet Columns: ["column1" (0)] Values: [(1); (2)] | |--CPhysicalComputeScalar | | |--CPhysicalConstTableGet | | +--CScalarProjectList | | |--CScalarProjectElement "ColRef_0004" (4) | | | +--CScalarConst (1) | | +--CScalarProjectElement "generate_series" (2) | | +--CScalarFunc (generate_series) | | |--CScalarConst (2) | | +--CScalarConst (3) | +--CScalarCmp (=) | |--CScalarConst (3) | +--CScalarIdent "generate_series" (2) +--CScalarProjectList +--CScalarProjectElement "?column?" (3) +--CScalarIdent "ColRef_0004" (4) ``` With predicate push down: ``` Physical plan: +--CPhysicalComputeScalar |--CPhysicalCorrelatedLeftOuterNLJoin | |--CPhysicalConstTableGet Columns: ["column1" (0)] Values: [(1); (2)] | |--CPhysicalComputeScalar | | |--CPhysicalFilter | | | |--CPhysicalComputeScalar | | | | |--CPhysicalConstTableGet | | | | +--CScalarProjectList | | | | +--CScalarProjectElement "generate_series" (2) | | | | +--CScalarFunc (generate_series) | | | | |--CScalarConst (2) | | | | +--CScalarConst (3) | | | +--CScalarCmp (=) | | | |--CScalarConst (3) | | | +--CScalarIdent "generate_series" (2) | | +--CScalarProjectList | | +--CScalarProjectElement "ColRef_0004" (4) | | +--CScalarConst (1) | +--CScalarConst (1) +--CScalarProjectList +--CScalarProjectElement "?column?" (3) +--CScalarIdent "ColRef_0004" (4) ``` Both of these plans result in a SUBPLAN node of type ALL_SUBLINK. Inside ExecScanSubPlan() executor will check that all evaulated expressions are true. Issue is that if we push the predicate down and filter out all the expressions that evaluate to false, then in example query we would incorrectly return true from ExecSubPlan(). Fix is to avoid pushing down the predicate in the case of SUBPLAN of type ALL_SUBLINK. --- .../CLogicalLeftOuterCorrelatedApply.h | 8 ++++ .../CLogicalLeftOuterCorrelatedApply.cpp | 44 ++++++++++++++++++- .../libgpopt/src/operators/CNormalizer.cpp | 14 +++++- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h index 7f3ed6b959f..63a3c0bc497 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h @@ -30,6 +30,8 @@ namespace gpopt class CLogicalLeftOuterCorrelatedApply : public CLogicalLeftOuterApply { private: + BOOL m_allow_predicate_pushdown{true}; + public: CLogicalLeftOuterCorrelatedApply(const CLogicalLeftOuterCorrelatedApply &) = delete; @@ -77,6 +79,12 @@ class CLogicalLeftOuterCorrelatedApply : public CLogicalLeftOuterApply return true; } + BOOL + IsPredicatePushDownAllowed() const + { + return m_allow_predicate_pushdown; + } + // conversion function static CLogicalLeftOuterCorrelatedApply * PopConvert(COperator *pop) diff --git a/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp b/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp index 400442e8b01..7f9d1429c62 100644 --- a/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp @@ -40,7 +40,49 @@ CLogicalLeftOuterCorrelatedApply::CLogicalLeftOuterCorrelatedApply( //--------------------------------------------------------------------------- CLogicalLeftOuterCorrelatedApply::CLogicalLeftOuterCorrelatedApply( CMemoryPool *mp, CColRefArray *pdrgpcrInner, EOperatorId eopidOriginSubq) - : CLogicalLeftOuterApply(mp, pdrgpcrInner, eopidOriginSubq) + : CLogicalLeftOuterApply(mp, pdrgpcrInner, eopidOriginSubq), + // In the case of subquery all, we cannot push down the predicate. + // Example query: + // + // SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); + // + // Physical plan: + // ``` + // +--CPhysicalComputeScalar + // |--CPhysicalCorrelatedLeftOuterNLJoin + // | |--CPhysicalConstTableGet Columns: ["" (0)] + // | |--CPhysicalComputeScalar + // | | |--CPhysicalCorrelatedLeftOuterNLJoin + // | | | |--CPhysicalComputeScalar + // | | | | |--CPhysicalConstTableGet Columns: ["" (1)] Values: [(1)] + // | | | | +--CScalarProjectList + // | | | | +--CScalarProjectElement "generate_series" (2) + // | | | | +--CScalarFunc (generate_series) + // | | | | |--CScalarConst (1) + // | | | | +--CScalarConst (2) + // | | | |--CPhysicalComputeScalar + // | | | | |--CPhysicalConstTableGet Columns: ["" (3)] Values: [(1)] + // | | | | +--CScalarProjectList + // | | | | +--CScalarProjectElement "?column?" (4) + // | | | | +--CScalarConst (1) + // | | | +--CScalarConst (1) + // | | +--CScalarProjectList + // | | +--CScalarProjectElement "ColRef_0006" (6) + // | | +--CScalarConst (1) + // | +--CScalarCmp (=) + // | |--CScalarIdent "?column?" (4) + // | +--CScalarIdent "generate_series" (2) + // +--CScalarProjectList + // +--CScalarProjectElement "?column?" (5) + // +--CScalarIdent "ColRef_0006" (6) + // ``` + // + // If we push down CScalarCmp as a filter then we would incorrectly + // discard the tuple 2 output from generate_series. Instead we want to + // preserve it for a NULL match in the LOJ so that we correctly evaulate + // subplan ALL_SUBLINK. + m_allow_predicate_pushdown(COperator::EopScalarSubqueryAll != + eopidOriginSubq) { } diff --git a/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp b/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp index 9739ab89857..4b179a90ba3 100644 --- a/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp @@ -18,6 +18,7 @@ #include "gpopt/base/CUtils.h" #include "gpopt/operators/CLogical.h" #include "gpopt/operators/CLogicalInnerJoin.h" +#include "gpopt/operators/CLogicalLeftOuterCorrelatedApply.h" #include "gpopt/operators/CLogicalLeftOuterJoin.h" #include "gpopt/operators/CLogicalNAryJoin.h" #include "gpopt/operators/CLogicalProject.h" @@ -1078,7 +1079,6 @@ CNormalizer::PushThru(CMemoryPool *mp, CExpression *pexprLogical, case COperator::EopLogicalInnerCorrelatedApply: case COperator::EopLogicalLeftOuterJoin: case COperator::EopLogicalLeftOuterApply: - case COperator::EopLogicalLeftOuterCorrelatedApply: case COperator::EopLogicalLeftSemiApply: case COperator::EopLogicalLeftSemiApplyIn: case COperator::EopLogicalLeftSemiCorrelatedApplyIn: @@ -1088,9 +1088,19 @@ CNormalizer::PushThru(CMemoryPool *mp, CExpression *pexprLogical, case COperator::EopLogicalLeftSemiJoin: PushThruJoin(mp, pexprLogical, pexprConj, ppexprResult); break; - + case COperator::EopLogicalLeftOuterCorrelatedApply: default: { + if (COperator::EopLogicalLeftOuterCorrelatedApply == + pexprLogical->Pop()->Eopid() && + CLogicalLeftOuterCorrelatedApply::PopConvert( + pexprLogical->Pop()) + ->IsPredicatePushDownAllowed()) + { + PushThruJoin(mp, pexprLogical, pexprConj, ppexprResult); + break; + } + // can't push predicates through, start a new normalization path CExpression *pexprNormalized = PexprRecursiveNormalize(mp, pexprLogical); From 61a8a57985f7b0fac3fe2f51ba725a48cfd28339 Mon Sep 17 00:00:00 2001 From: David Kimura Date: Mon, 31 Jan 2022 20:00:01 +0000 Subject: [PATCH 06/24] Allow subplan test to contain scalar ident as left expression In order to support subplan test expressions with scalar ident params on the left side of the test expression, we must allow more than one column to be projected from the restricted result node beneath. --- .../translate/CTranslatorDXLToScalar.cpp | 43 +++++++++--- .../libgpopt/include/gpopt/base/CColRefSet.h | 3 + .../gpopt/translate/CTranslatorExprToDXL.h | 5 +- .../gporca/libgpopt/src/base/CColRefSet.cpp | 24 ++++++- .../src/translate/CTranslatorExprToDXL.cpp | 69 +++++++++++++++---- .../dxl/operators/CDXLScalarSubPlan.h | 11 ++- .../src/operators/CDXLScalarSubPlan.cpp | 6 +- .../gpopt/translate/CTranslatorDXLToScalar.h | 1 + 8 files changed, 137 insertions(+), 25 deletions(-) diff --git a/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp b/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp index 6d87816dca4..331650e65bc 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp @@ -806,7 +806,8 @@ CTranslatorDXLToScalar::TranslateDXLScalarSubplanToScalar( SubLinkType slink = CTranslatorUtils::MapDXLSubplanToSublinkType( dxlop->GetDxlSubplanType()); Expr *test_expr = TranslateDXLSubplanTestExprToScalar( - dxlop->GetDxlTestExpr(), slink, colid_var, ¶m_ids); + dxlop->GetDxlTestExpr(), slink, colid_var, dxlop->FOuterParam(), + ¶m_ids); const CDXLColRefArray *outer_refs = dxlop->GetDxlOuterColRefsArray(); @@ -890,7 +891,7 @@ CTranslatorDXLToScalar::TranslateDXLScalarSubplanToScalar( Expr * CTranslatorDXLToScalar::TranslateDXLSubplanTestExprToScalar( CDXLNode *test_expr_node, SubLinkType slink, CMappingColIdVar *colid_var, - List **param_ids) + BOOL has_outer_refs, List **param_ids) { if (EXPR_SUBLINK == slink || EXISTS_SUBLINK == slink || NOT_EXISTS_SUBLINK == slink) @@ -923,19 +924,45 @@ CTranslatorDXLToScalar::TranslateDXLSubplanTestExprToScalar( CDXLNode *outer_child_node = (*test_expr_node)[0]; CDXLNode *inner_child_node = (*test_expr_node)[1]; + CContextDXLToPlStmt *dxl_to_plstmt_ctxt = + (dynamic_cast(colid_var)) + ->GetDXLToPlStmtContext(); + // translate outer expression (can be a deep scalar tree) - Expr *outer_arg_expr = TranslateDXLToScalar(outer_child_node, colid_var); - args = gpdb::LAppend(args, outer_arg_expr); + Expr *outer_arg_expr = nullptr; + if (has_outer_refs) + { + Param *param1 = MakeNode(Param); + param1->paramkind = PARAM_EXEC; + + // Ident + CDXLScalarIdent *outer_ident = + CDXLScalarIdent::Cast(outer_child_node->GetOperator()); + Expr *outer_expr = (Expr *) param1; + + // finalize outer expression + param1->paramtype = CMDIdGPDB::CastMdid(outer_ident->MdidType())->Oid(); + param1->paramtypmod = outer_ident->TypeModifier(); + param1->paramid = dxl_to_plstmt_ctxt->GetNextParamId(param1->paramtype); + + // test expression is used for non-scalar subplan, + // first arg of test expression must be an EXEC param1 referring to subplan output + args = gpdb::LAppend(args, outer_expr); + + // also, add this param1 to subplan param1 ids before translating other params + *param_ids = gpdb::LAppendInt(*param_ids, param1->paramid); + } + else + { + outer_arg_expr = TranslateDXLToScalar(outer_child_node, colid_var); + args = gpdb::LAppend(args, outer_arg_expr); + } // translate inner expression (only certain forms supported) // second arg must be an EXEC param which is replaced during query execution with subplan output Param *param = MakeNode(Param); param->paramkind = PARAM_EXEC; - CContextDXLToPlStmt *dxl_to_plstmt_ctxt = - (dynamic_cast(colid_var)) - ->GetDXLToPlStmtContext(); - CDXLScalarIdent *inner_ident = nullptr; Expr *inner_expr = nullptr; if (EdxlopScalarIdent == inner_child_node->GetOperator()->GetDXLOperator()) diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h b/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h index 3a38ec04f17..007d6671cff 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h @@ -120,6 +120,9 @@ class CColRefSet : public CBitSet, public DbgPrintMixin // convert to array CColRefArray *Pdrgpcr(CMemoryPool *mp) const; + // convert to id colref map + IntToColRefMap *Phmicr(CMemoryPool *mp) const; + // hash function ULONG HashValue(); diff --git a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h index 269e4ec2fa6..aeea672aa96 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h +++ b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h @@ -625,7 +625,10 @@ class CTranslatorExprToDXL CDXLNode *PdxlnProjectBoolConst(CDXLNode *dxlnode, BOOL value); // helper to build a Result expression with project list restricted to required column - CDXLNode *PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref); + CDXLNode *PdxlnRestrictResult(CDXLNode *dxlnode, const CColRef *colref); + + // helper to build a Result expression with project list restricted to required columns + CDXLNode *PdxlnRestrictResult(CDXLNode *dxlnode, const CColRefSet *colrefs); // helper to build subplans from correlated LOJ void BuildSubplansForCorrelatedLOJ( diff --git a/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp b/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp index a3ba3ee37bc..716e071958b 100644 --- a/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp +++ b/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp @@ -88,7 +88,6 @@ CColRefSet::FMember(const CColRef *colref) const return CBitSet::Get(colref->Id()); } - //--------------------------------------------------------------------------- // @function: // CColRefSet::PcrAny @@ -294,6 +293,29 @@ CColRefSet::Pdrgpcr(CMemoryPool *mp) const } +//--------------------------------------------------------------------------- +// @function: +// CColRefSet::Phmicr +// +// @doc: +// Convert set into map +// +//--------------------------------------------------------------------------- +IntToColRefMap * +CColRefSet::Phmicr(CMemoryPool *mp) const +{ + IntToColRefMap *phmicr = GPOS_NEW(mp) IntToColRefMap(mp); + + CColRefSetIter crsi(*this); + while (crsi.Advance()) + { + phmicr->Insert(GPOS_NEW(mp) INT(crsi.Pcr()->Id()), crsi.Pcr()); + } + + return phmicr; +} + + //--------------------------------------------------------------------------- // @function: // CColRefSet::HashValue diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp index ae35120faaf..a3d421b8cd8 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp @@ -3317,10 +3317,35 @@ CTranslatorExprToDXL::BuildSubplans( // //--------------------------------------------------------------------------- CDXLNode * -CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref) +CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, + const CColRef *colref) +{ + CDXLNode *dxlresult = nullptr; + CColRefSet *pcrInner = GPOS_NEW(m_mp) CColRefSet(m_mp); + + pcrInner->Include(colref); + dxlresult = PdxlnRestrictResult(dxlnode, pcrInner); + pcrInner->Release(); + + return dxlresult; +} + + +//--------------------------------------------------------------------------- +// @function: +// CTranslatorExprToDXL::PdxlnRestrictResult +// +// @doc: +// Helper to build a Result expression with project list +// restricted to required columns +// +//--------------------------------------------------------------------------- +CDXLNode * +CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, + const CColRefSet *colrefs) { GPOS_ASSERT(nullptr != dxlnode); - GPOS_ASSERT(nullptr != colref); + GPOS_ASSERT(nullptr != colrefs); CDXLNode *pdxlnProjListOld = (*dxlnode)[0]; const ULONG ulPrjElems = pdxlnProjListOld->Arity(); @@ -3339,12 +3364,17 @@ CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref) CDXLScalarProjList *pdxlopPrL = GPOS_NEW(m_mp) CDXLScalarProjList(m_mp); CDXLNode *pdxlnProjListNew = GPOS_NEW(m_mp) CDXLNode(m_mp, pdxlopPrL); + IntToColRefMap *phmicr = colrefs->Phmicr(m_mp); + for (ULONG ul = 0; ul < ulPrjElems; ul++) { CDXLNode *child_dxlnode = (*pdxlnProjListOld)[ul]; CDXLScalarProjElem *pdxlPrjElem = CDXLScalarProjElem::Cast(child_dxlnode->GetOperator()); - if (pdxlPrjElem->Id() == colref->Id()) + + const INT colid = pdxlPrjElem->Id(); + CColRef *colref = phmicr->Find(&colid); + if (colref) { // create a new project element that simply points to required column, // we cannot re-use child_dxlnode here since it may have a deep expression with columns inaccessible @@ -3354,7 +3384,10 @@ CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref) pdxlnProjListNew->AddChild(pdxlnPrEl); } } - GPOS_ASSERT(1 == pdxlnProjListNew->Arity()); + + phmicr->Release(); + + GPOS_ASSERT(colrefs->Size() == pdxlnProjListNew->Arity()); pdxlnResult = GPOS_NEW(m_mp) CDXLNode(m_mp, GPOS_NEW(m_mp) CDXLPhysicalResult(m_mp)); @@ -3409,8 +3442,10 @@ CTranslatorExprToDXL::PdxlnQuantifiedSubplan( pulNonGatherMotions, pfDML, false /*fRemap*/, false /*fRoot*/); // find required column from inner child - CColRef *pcrInner = (*pdrgpcrInner)[0]; + CColRefSet *pcrInner = GPOS_NEW(m_mp) CColRefSet(m_mp); + pcrInner->Include((*pdrgpcrInner)[0]); + BOOL outerParam = false; if (fCorrelatedLOJ) { // overwrite required inner column based on scalar expression @@ -3421,14 +3456,24 @@ CTranslatorExprToDXL::PdxlnQuantifiedSubplan( pcrsUsed->Intersection(pcrsInner); if (0 < pcrsUsed->Size()) { - GPOS_ASSERT(1 == pcrsUsed->Size()); + GPOS_ASSERT(1 == pcrsUsed->Size() || 2 == pcrsUsed->Size()); + + // Both sides of the SubPlan test expression can come from the + // inner side. So we need to pass pcrsUsed instead of pcrInner into + // PdxlnRestrictResult() + outerParam = pcrsUsed->Size() > 1; - pcrInner = pcrsUsed->PcrFirst(); + pcrInner->Release(); + pcrInner = pcrsUsed; + } + else + { + pcrsUsed->Release(); } - pcrsUsed->Release(); } CDXLNode *inner_dxlnode = PdxlnRestrictResult(pdxlnInnerChild, pcrInner); + pcrInner->Release(); if (nullptr == inner_dxlnode) { GPOS_RAISE( @@ -3445,10 +3490,10 @@ CTranslatorExprToDXL::PdxlnQuantifiedSubplan( mdid->AddRef(); // construct a subplan node, with the inner child under it - CDXLNode *pdxlnSubPlan = GPOS_NEW(m_mp) CDXLNode( - m_mp, - GPOS_NEW(m_mp) CDXLScalarSubPlan(m_mp, mdid, dxl_colref_array, - dxl_subplan_type, dxlnode_test_expr)); + CDXLNode *pdxlnSubPlan = GPOS_NEW(m_mp) + CDXLNode(m_mp, GPOS_NEW(m_mp) CDXLScalarSubPlan( + m_mp, mdid, dxl_colref_array, dxl_subplan_type, + dxlnode_test_expr, outerParam)); pdxlnSubPlan->AddChild(inner_dxlnode); // add to hashmap diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h index 54855430eec..00518f7b955 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h @@ -63,6 +63,9 @@ class CDXLScalarSubPlan : public CDXLScalar // test expression -- not null if quantified/existential subplan CDXLNode *m_dxlnode_test_expr; + // does test expression contain outer param + BOOL m_outer_param; + public: CDXLScalarSubPlan(CDXLScalarSubPlan &) = delete; @@ -70,7 +73,7 @@ class CDXLScalarSubPlan : public CDXLScalar CDXLScalarSubPlan(CMemoryPool *mp, IMDId *first_col_type_mdid, CDXLColRefArray *dxl_colref_array, EdxlSubPlanType dxl_subplan_type, - CDXLNode *dxlnode_test_expr); + CDXLNode *dxlnode_test_expr, BOOL outer_param = false); ~CDXLScalarSubPlan() override; @@ -108,6 +111,12 @@ class CDXLScalarSubPlan : public CDXLScalar return m_dxlnode_test_expr; } + BOOL + FOuterParam() const + { + return m_outer_param; + } + // serialize operator in DXL format void SerializeToDXL(CXMLSerializer *xml_serializer, const CDXLNode *dxlnode) const override; diff --git a/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp b/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp index 42c4bb4fe37..6274d9236a4 100644 --- a/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp +++ b/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp @@ -33,12 +33,14 @@ CDXLScalarSubPlan::CDXLScalarSubPlan(CMemoryPool *mp, IMDId *first_col_type_mdid, CDXLColRefArray *dxl_colref_array, EdxlSubPlanType dxl_subplan_type, - CDXLNode *dxlnode_test_expr) + CDXLNode *dxlnode_test_expr, + BOOL outer_param) : CDXLScalar(mp), m_first_col_type_mdid(first_col_type_mdid), m_dxl_colref_array(dxl_colref_array), m_dxl_subplan_type(dxl_subplan_type), - m_dxlnode_test_expr(dxlnode_test_expr) + m_dxlnode_test_expr(dxlnode_test_expr), + m_outer_param(outer_param) { GPOS_ASSERT(EdxlSubPlanTypeSentinel > dxl_subplan_type); GPOS_ASSERT_IMP(EdxlSubPlanTypeAny == dxl_subplan_type || diff --git a/src/include/gpopt/translate/CTranslatorDXLToScalar.h b/src/include/gpopt/translate/CTranslatorDXLToScalar.h index e4fa180b69f..bd5672ccaea 100644 --- a/src/include/gpopt/translate/CTranslatorDXLToScalar.h +++ b/src/include/gpopt/translate/CTranslatorDXLToScalar.h @@ -150,6 +150,7 @@ class CTranslatorDXLToScalar Expr *TranslateDXLSubplanTestExprToScalar(CDXLNode *test_expr_node, SubLinkType slink, CMappingColIdVar *colid_var, + BOOL has_outer_refs, List **param_ids_list); // translate subplan parameters From cef1d1c2b6402fa468b7df1f2b8ff2313ac857c7 Mon Sep 17 00:00:00 2001 From: David Kimura Date: Thu, 27 Jan 2022 23:40:31 +0000 Subject: [PATCH 07/24] Add correlated subquery testcases --- .../AnyPredicate-Over-UnionOfConsts.mdp | 4 + .../dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp | 332 ++++++++++++++++ .../dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp | 360 ++++++++++++++++++ .../gpopt/xforms/CSubqueryHandlerTest.cpp | 4 +- .../regress/expected/correlated_subquery.out | 128 +++++++ src/test/regress/greenplum_schedule | 2 +- src/test/regress/sql/correlated_subquery.sql | 21 + 7 files changed, 849 insertions(+), 2 deletions(-) create mode 100644 src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp create mode 100644 src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp create mode 100644 src/test/regress/expected/correlated_subquery.out create mode 100644 src/test/regress/sql/correlated_subquery.sql diff --git a/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp b/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp index cb496402c58..66786c38994 100644 --- a/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp +++ b/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp @@ -1,5 +1,9 @@ + + diff --git a/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp new file mode 100644 index 00000000000..2fbe8cfa699 --- /dev/null +++ b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp @@ -0,0 +1,332 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp new file mode 100644 index 00000000000..6a9b0fdb942 --- /dev/null +++ b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp @@ -0,0 +1,360 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp b/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp index 0c730984807..8b87f813205 100644 --- a/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp +++ b/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp @@ -34,7 +34,9 @@ const CHAR *rgszSubqueryHandlerMinidumpFileNames[] = { "../data/dxl/minidump/CorrelatedSubqueryWithAggWindowFunc.mdp", "../data/dxl/minidump/AllSubqueryWithSubqueryInScalar.mdp", "../data/dxl/minidump/AnySubqueryWithAllSubqueryInScalar.mdp", - "../data/dxl/minidump/AnySubqueryWithSubqueryInScalar.mdp"}; + "../data/dxl/minidump/AnySubqueryWithSubqueryInScalar.mdp", + "../data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp", + "../data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp"}; //--------------------------------------------------------------------------- // @function: diff --git a/src/test/regress/expected/correlated_subquery.out b/src/test/regress/expected/correlated_subquery.out new file mode 100644 index 00000000000..99625097746 --- /dev/null +++ b/src/test/regress/expected/correlated_subquery.out @@ -0,0 +1,128 @@ +SET optimizer_enforce_subplans = 1; +SET optimizer_trace_fallback=on; +SET client_min_messages=log; +SELECT a = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT a = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + f | 2 +(2 rows) + +SELECT a = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT a = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + t | 2 +(2 rows) + +SELECT 1 = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 1 = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT 2 = ALL (SELECT generate_series(2, 2)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2 = ALL (SELECT generate_series(2, 2)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT 2 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT 2+1 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2+1 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT 2+1 = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2+1 = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT (SELECT a) = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT a) = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + f | 2 +(2 rows) + +SELECT (SELECT a) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT a) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + t | 2 +(2 rows) + +SELECT (SELECT a+1) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT a+1) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + t | 1 + f | 2 +(2 rows) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)); + ?column? +---------- + t +(1 row) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); + ?column? +---------- + f +(1 row) + +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)); +LOG: statement: SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)); + ?column? +---------- + t +(1 row) + diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 71d0e97eaa1..c12d6ad33f5 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -191,7 +191,7 @@ test: qp_misc_jiras qp_with_clause qp_executor qp_olap_windowerr qp_olap_window test: qp_with_functional_inlining qp_with_functional_noinlining test: qp_functions_in_contexts_setup # test: qp_misc_rio_join_small qp_misc_rio qp_correlated_query qp_targeted_dispatch qp_gist_indexes2 qp_gist_indexes3 qp_gist_indexes4 qp_query_execution qp_functions_in_from qp_functions_in_select qp_functions_in_subquery qp_functions_in_subquery_column qp_functions_in_subquery_constant qp_functions_in_with -test: qp_misc_rio_join_small qp_correlated_query qp_targeted_dispatch qp_gist_indexes2 qp_gist_indexes3 qp_gist_indexes4 qp_query_execution qp_functions_in_from qp_functions_in_select qp_functions_in_subquery qp_functions_in_subquery_column qp_functions_in_with +test: qp_misc_rio_join_small qp_correlated_query qp_targeted_dispatch qp_gist_indexes2 qp_gist_indexes3 qp_gist_indexes4 qp_query_execution qp_functions_in_from qp_functions_in_select qp_functions_in_subquery qp_functions_in_subquery_column qp_functions_in_with correlated_subquery test: dpe qp_dpe qp_subquery qp_left_anti_semi_join qp_union_intersect qp_functions qp_functions_idf qp_regexp qp_resource_queue qp_orca_fallback diff --git a/src/test/regress/sql/correlated_subquery.sql b/src/test/regress/sql/correlated_subquery.sql new file mode 100644 index 00000000000..b4a6d8283c3 --- /dev/null +++ b/src/test/regress/sql/correlated_subquery.sql @@ -0,0 +1,21 @@ +SET optimizer_enforce_subplans = 1; +SET optimizer_trace_fallback=on; +SET client_min_messages=log; + +SELECT a = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +SELECT a = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +SELECT 1 = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +SELECT 2 = ALL (SELECT generate_series(2, 2)) FROM (values (1),(2)) v(a); +SELECT 2 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +SELECT 2+1 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +SELECT 2+1 = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); +SELECT (SELECT a) = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +SELECT (SELECT a) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +SELECT (SELECT a+1) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)) FROM (values (1),(2)) v(a); +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)); +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)); From 32a6f9494b0feb041efe165167989ac3e67b8f8e Mon Sep 17 00:00:00 2001 From: Alexandra Wang Date: Tue, 8 Feb 2022 16:43:09 +0800 Subject: [PATCH 08/24] GPORCA CI: add workload6 to explain pipeline --- .../concourse/test_explain_pipeline.yml | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/src/backend/gporca/concourse/test_explain_pipeline.yml b/src/backend/gporca/concourse/test_explain_pipeline.yml index 6d8a31da844..81759a2b82c 100644 --- a/src/backend/gporca/concourse/test_explain_pipeline.yml +++ b/src/backend/gporca/concourse/test_explain_pipeline.yml @@ -171,6 +171,28 @@ resources: json_key: ((concourse-gcs-resources-orca-service-account-key)) versioned_file: ((pipeline-name))/explain_results/explain_((workload4))_results.tar.gz +## resources for workload 6 +- name: explain_output_((workload6))_branch + type: gcs + source: + bucket: ((gcs-bucket-orca)) + json_key: ((concourse-gcs-resources-orca-service-account-key)) + versioned_file: ((pipeline-name))/explain_intermediate/explain_((workload6))_branch.tar.gz + +- name: explain_output_((workload6))_baseline + type: gcs + source: + bucket: ((gcs-bucket-orca)) + json_key: ((concourse-gcs-resources-orca-service-account-key)) + versioned_file: ((pipeline-name))/explain_intermediate/explain_((workload6))_baseline.tar.gz + +- name: explain_((workload6))_results + type: gcs + source: + bucket: ((gcs-bucket-orca)) + json_key: ((concourse-gcs-resources-orca-service-account-key)) + versioned_file: ((pipeline-name))/explain_results/explain_((workload6))_results.tar.gz + ## ====================================================================== ## jobs ## ====================================================================== @@ -631,3 +653,94 @@ jobs: put: explain_((workload4))_results params: file: diffs/explain_test_results.tar.gz + +## ====================================================================== +## workload6 +## ====================================================================== + +- name: run_explain_suite_((workload6)) + max_in_flight: 1 + plan: + - in_parallel: + - get: gporca-commits-to-test + passed: + - compile_branch + - get: bin_gpdb + passed: + - compile_branch + resource: bin_gpdb_centos7_branch + trigger: true + - get: gpdb_src + resource: gpdb_main_src + - get: gp_workloads + params: + disable_git_lfs: true + depth: 1 + - task: run_explain_suite + file: gpdb_src/concourse/tasks/run_explain_suite.yml + params: + MODE: orca + WORKLOAD: ((workload6)) + on_success: + try: + put: explain_output_((workload6))_branch + params: + file: output/explain_ouput.tar.gz + +- name: run_explain_suite_((workload6))_baseline + max_in_flight: 1 + plan: + - in_parallel: + - get: gporca-commits-to-test + passed: + - compile_baseline + - get: bin_gpdb + resource: bin_gpdb_centos7_baseline + passed: + - compile_baseline + trigger: true + - get: gpdb_src + resource: gpdb_main_src + params: + submodules: none + - get: gp_workloads + params: + disable_git_lfs: true + depth: 1 + - task: run_explain_suite_baseline + file: gpdb_src/concourse/tasks/run_explain_suite.yml + params: + MODE: orca + WORKLOAD: ((workload6)) + on_success: + try: + put: explain_output_((workload6))_baseline + params: + file: output/explain_ouput.tar.gz + +- name: diff_explain_results_((workload6)) + plan: + - in_parallel: + - get: gporca-commits-to-test + passed: + - run_explain_suite_((workload6)) + - run_explain_suite_((workload6))_baseline + trigger: true + - get: gpdb_src + resource: gpdb_main_src + params: + submodules: none + - get: explain_output + resource: explain_output_((workload6))_branch + passed: + - run_explain_suite_((workload6)) + - get: explain_output_baseline + resource: explain_output_((workload6))_baseline + passed: + - run_explain_suite_((workload6))_baseline + - task: diff_explain_results_((workload6)) + file: gpdb_src/concourse/tasks/diff_explain_results_with_baseline.yml + ensure: + put: explain_((workload6))_results + params: + file: diffs/explain_test_results.tar.gz From be1117d628168ead19b5719f9be949df5e6f64cf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 15 Feb 2022 12:21:28 -0500 Subject: [PATCH 09/24] Improve subscriber's error message for wrong publication relkind. Pre-v13 versions only support logical replication from plain tables, while v13 and later also allow partitioned tables to be published. If you tried to subscribe an older server to such a publication, you got "table XXX not found on publisher", which is pretty unhelpful/confusing. Arrange to deliver a more on-point error message. As commit c314c147c did in v13, remove the relkind check from the query WHERE clause altogether, so that "not there" is distinguishable from "wrong relkind". Per report from Radoslav Nedyalkov. Patch v10-v12. Discussion: https://postgr.es/m/2952568.1644876730@sss.pgh.pa.us --- src/backend/replication/logical/tablesync.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 4c6f8646cb8..d104ccf86e1 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -700,9 +700,10 @@ fetch_remote_table_info(char *nspname, char *relname, WalRcvExecResult *res; StringInfoData cmd; TupleTableSlot *slot; - Oid tableRow[] = {OIDOID, CHAROID, CHAROID}; - Oid attrRow[] = {TEXTOID, OIDOID, BOOLOID}; + Oid tableRow[3] = {OIDOID, CHAROID, CHAROID}; + Oid attrRow[4] = {TEXTOID, OIDOID, INT4OID, BOOLOID}; bool isnull; + char relkind; int natt; lrel->nspname = nspname; @@ -738,9 +739,20 @@ fetch_remote_table_info(char *nspname, char *relname, Assert(!isnull); lrel->replident = DatumGetChar(slot_getattr(slot, 2, &isnull)); Assert(!isnull); - lrel->relkind = DatumGetChar(slot_getattr(slot, 3, &isnull)); + relkind = DatumGetChar(slot_getattr(slot, 3, &isnull)); Assert(!isnull); + /* + * Newer PG versions allow things that aren't plain tables to appear in + * publications. We don't handle that in this version, but try to provide + * a useful error message. + */ + if (relkind != RELKIND_RELATION) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("logical replication source relation \"%s.%s\" is not a table", + nspname, relname))); + ExecDropSingleTupleTableSlot(slot); walrcv_clear_result(res); From be2d5571632da04744b3face5d8135916306b89a Mon Sep 17 00:00:00 2001 From: Chris Hajas Date: Tue, 8 Feb 2022 15:33:38 -0800 Subject: [PATCH 10/24] Fix flaky aggregates test Sometimes these prepared statements would be replanned and log ORCA fallbacks, which are expected and ok. However, to make this more deterministic, reset the plan cache in the test. --- src/test/regress/expected/aggregates.out | 2 ++ .../regress/expected/aggregates_optimizer.out | 22 +++++++++++++++++-- src/test/regress/sql/aggregates.sql | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index c47e03dfd88..62548a4c69c 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -2773,6 +2773,8 @@ create aggregate my_avg_init2(int4) finalfunc = avg_finalfn, initcond = '(4,0)' ); +-- reset the plan cache, sometimes it would re-plan these prepared statements and log ORCA fallbacks +discard plans; -- state should be shared if INITCONDs are matching select my_sum_init(one),my_avg_init(one) from (values(1),(3)) t(one); NOTICE: avg_transfn called with 1 diff --git a/src/test/regress/expected/aggregates_optimizer.out b/src/test/regress/expected/aggregates_optimizer.out index 33b127d375e..634744680d9 100644 --- a/src/test/regress/expected/aggregates_optimizer.out +++ b/src/test/regress/expected/aggregates_optimizer.out @@ -2929,12 +2929,30 @@ create aggregate my_avg_init2(int4) finalfunc = avg_finalfn, initcond = '(4,0)' ); +-- reset the plan cache, sometimes it would re-plan these prepared statements and log ORCA fallbacks +discard plans; -- state should be shared if INITCONDs are matching select my_sum_init(one),my_avg_init(one) from (values(1),(3)) t(one); +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL NOTICE: avg_transfn called with 1 -NOTICE: avg_transfn called with 1 -NOTICE: avg_transfn called with 3 +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL NOTICE: avg_transfn called with 3 +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL my_sum_init | my_avg_init -------------+------------- 14 | 7 diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql index 9268a11b3df..16b811d9bbe 100644 --- a/src/test/regress/sql/aggregates.sql +++ b/src/test/regress/sql/aggregates.sql @@ -1103,6 +1103,8 @@ create aggregate my_avg_init2(int4) initcond = '(4,0)' ); +-- reset the plan cache, sometimes it would re-plan these prepared statements and log ORCA fallbacks +discard plans; -- state should be shared if INITCONDs are matching select my_sum_init(one),my_avg_init(one) from (values(1),(3)) t(one); From 176a5bd3d93d9d16a6da3ce2c781e7142738b463 Mon Sep 17 00:00:00 2001 From: Chris Hajas Date: Tue, 8 Feb 2022 15:57:51 -0800 Subject: [PATCH 11/24] Enable optimizer_trace_fallback in gp_dqa ICW test We saw a flake here, but haven't been able to repro. It's a fallback to planner, but we're not exactly sure why. Hopefully this will give us a hint if it happens again, and it's good to enable this so we can more easily tell when and why Orca falls back. --- src/test/regress/expected/gp_dqa.out | 1 + .../regress/expected/gp_dqa_optimizer.out | 149 ++++++++++++++++++ src/test/regress/sql/gp_dqa.sql | 2 + 3 files changed, 152 insertions(+) diff --git a/src/test/regress/expected/gp_dqa.out b/src/test/regress/expected/gp_dqa.out index 36643b10471..110611f01c5 100644 --- a/src/test/regress/expected/gp_dqa.out +++ b/src/test/regress/expected/gp_dqa.out @@ -4,6 +4,7 @@ -- differences by setting 'extra_float_digits'. This isn't enough for all of -- the queries, so a few also use TO_CHAR() to truncate the results further. set extra_float_digits=0; +SET optimizer_trace_fallback to on; drop table if exists dqa_t1; NOTICE: table "dqa_t1" does not exist, skipping drop table if exists dqa_t2; diff --git a/src/test/regress/expected/gp_dqa_optimizer.out b/src/test/regress/expected/gp_dqa_optimizer.out index 3897ae46f1b..bcc734b3978 100644 --- a/src/test/regress/expected/gp_dqa_optimizer.out +++ b/src/test/regress/expected/gp_dqa_optimizer.out @@ -4,6 +4,7 @@ -- differences by setting 'extra_float_digits'. This isn't enough for all of -- the queries, so a few also use TO_CHAR() to truncate the results further. set extra_float_digits=0; +SET optimizer_trace_fallback to on; drop table if exists dqa_t1; NOTICE: table "dqa_t1" does not exist, skipping drop table if exists dqa_t2; @@ -114,12 +115,16 @@ explain (costs off) select count(distinct d), sum(distinct d) from dqa_t1 group (11 rows) select count(distinct d), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 23 | 34 (1 row) explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------ Finalize Aggregate @@ -136,12 +141,16 @@ explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1; (11 rows) select count(distinct d), count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | count -------+-------+------- 23 | 10 | 34 (1 row) explain (costs off) select count(distinct d), count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------ Finalize Aggregate @@ -158,6 +167,8 @@ explain (costs off) select count(distinct d), count(distinct c), count(distinct (11 rows) select count(distinct d), count(distinct dt) from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 10 | 10 @@ -173,6 +184,8 @@ select count(distinct d), count(distinct dt) from dqa_t1 group by c; (10 rows) explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -194,6 +207,8 @@ explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1 gro (16 rows) select count(distinct d), count(distinct dt) from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 1 | 5 @@ -222,6 +237,8 @@ select count(distinct d), count(distinct dt) from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -483,12 +500,16 @@ explain (costs off) select count(distinct i), sum(distinct i) from dqa_t1 group (9 rows) select count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 10 | 34 (1 row) explain (costs off) select count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------ Finalize Aggregate @@ -505,6 +526,8 @@ explain (costs off) select count(distinct c), count(distinct dt) from dqa_t1; (11 rows) select count(distinct c), count(distinct dt), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | i -------+-------+---- 5 | 9 | 3 @@ -522,6 +545,8 @@ select count(distinct c), count(distinct dt), i from dqa_t1 group by i; (12 rows) explain (costs off) select count(distinct c), count(distinct dt), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -543,6 +568,8 @@ explain (costs off) select count(distinct c), count(distinct dt), i from dqa_t1 (16 rows) select count(distinct i), count(distinct c), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | d -------+-------+---- 5 | 5 | 3 @@ -571,6 +598,8 @@ select count(distinct i), count(distinct c), d from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct i), count(distinct c), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -702,6 +731,8 @@ explain (costs off) select count(distinct dqa_t1.dt) from dqa_t1, dqa_t2 where d -- multidqa with groupby and order by select sum(distinct d), count(distinct i), count(distinct c),i,c from dqa_t1 group by i,c order by i,c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | count | count | i | c -----+-------+-------+----+--- 14 | 1 | 1 | 0 | 0 @@ -767,6 +798,8 @@ select sum(distinct d), count(distinct i), count(distinct c),i,c from dqa_t1 gro (60 rows) explain (costs off) select sum(distinct d), count(distinct i), count(distinct c),i,c from dqa_t1 group by i,c order by i,c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------ Sort @@ -877,12 +910,16 @@ explain (costs off) select to_char(corr(distinct d, i), '9.99999999999999') from -- multi args multidqa select count(distinct c), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr -------+-------------------- 10 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct c), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -899,12 +936,16 @@ explain (costs off) select count(distinct c), corr(distinct d, i) from dqa_t1; (11 rows) select count(distinct d), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr -------+-------------------- 23 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct d), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -921,12 +962,16 @@ explain (costs off) select count(distinct d), corr(distinct d, i) from dqa_t1; (11 rows) select count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | corr -------+-------+-------------------- 23 | 12 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -943,12 +988,16 @@ explain (costs off) select count(distinct d), count(distinct i), corr(distinct d (11 rows) select count(distinct c), count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | count | corr -------+-------+-------+-------------------- 10 | 23 | 12 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct c), count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------------------ Finalize Aggregate @@ -966,6 +1015,8 @@ explain (costs off) select count(distinct c), count(distinct d), count(distinct -- multi args multidqa with group by select count(distinct c), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | d -------+------+---- 5 | | 0 @@ -994,6 +1045,8 @@ select count(distinct c), corr(distinct d, i), d from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct c), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1015,6 +1068,8 @@ explain (costs off) select count(distinct c), corr(distinct d, i), d from dqa_t1 (16 rows) select count(distinct c), corr(distinct d, i), d, i from dqa_t1 group by d,i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | d | i -------+------+----+---- 1 | | 0 | 0 @@ -1120,6 +1175,8 @@ select count(distinct c), corr(distinct d, i), d, i from dqa_t1 group by d,i; (100 rows) explain (costs off) select count(distinct c), corr(distinct d, i), d, i from dqa_t1 group by d,i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1141,6 +1198,8 @@ explain (costs off) select count(distinct c), corr(distinct d, i), d, i from dqa (16 rows) select count(distinct c), corr(distinct d, i), dt from dqa_t1 group by dt; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | dt -------+--------------------+------------ 3 | 0.59603956067927 | 06-25-2009 @@ -1180,6 +1239,8 @@ select count(distinct c), corr(distinct d, i), dt from dqa_t1 group by dt; (34 rows) explain (costs off) select count(distinct c), corr(distinct d, i), dt from dqa_t1 group by dt; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN --------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1201,6 +1262,8 @@ explain (costs off) select count(distinct c), corr(distinct d, i), dt from dqa_t (16 rows) select count(distinct d), corr(distinct d, i), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | i -------+------+---- 9 | | 0 @@ -1218,6 +1281,8 @@ select count(distinct d), corr(distinct d, i), i from dqa_t1 group by i; (12 rows) explain (costs off) select count(distinct d), corr(distinct d, i), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1239,6 +1304,8 @@ explain (costs off) select count(distinct d), corr(distinct d, i), i from dqa_t1 (16 rows) select count(distinct d), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | d -------+------+---- 1 | | 0 @@ -1267,6 +1334,8 @@ select count(distinct d), corr(distinct d, i), d from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct d), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1288,6 +1357,8 @@ explain (costs off) select count(distinct d), corr(distinct d, i), d from dqa_t1 (16 rows) select count(distinct d), to_char(corr(distinct d, i), '9.99999999999999'), c from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | to_char | c -------+-------------------+--- 10 | .13670602618479 | 0 @@ -1303,6 +1374,8 @@ select count(distinct d), to_char(corr(distinct d, i), '9.99999999999999'), c f (10 rows) explain (costs off) select count(distinct d), to_char(corr(distinct d, i), '9.99999999999999'), c from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1359,6 +1432,8 @@ from fact_route_aggregation T218094 where ( T43883.device_id = T218094.device_id ) group by T43883.platform; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 ----+----+----+----+----+----+----+----+---- (0 rows) @@ -1384,6 +1459,8 @@ insert into t2_mdqa select i % 10 , i % 5, i || 'value' from generate_series(1, insert into t2_mdqa select i % 10 , i % 5, i || 'value' from generate_series(1, 20) i; -- simple mdqa select count(distinct t1.a), count(distinct t2.b), t1.c, t2.c from t1_mdqa t1, t2_mdqa t2 where t1.c = t2.c group by t1.c, t2.c order by t1.c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | c | c -------+-------+---------+--------- 1 | 1 | 10value | 10value @@ -1410,6 +1487,8 @@ select count(distinct t1.a), count(distinct t2.b), t1.c, t2.c from t1_mdqa t1, t -- distinct on top of some mdqas select distinct sum(distinct t1.a), avg(t2.a), sum(distinct t2.b), t1.a, t2.b from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.a, t2.b order by t1.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | avg | sum | a | b -----+------------------------+-----+---+--- 0 | 0.00000000000000000000 | 0 | 0 | 0 @@ -1420,6 +1499,8 @@ select distinct sum(distinct t1.a), avg(t2.a), sum(distinct t2.b), t1.a, t2.b fr (5 rows) select distinct sum (distinct t1.a), avg(distinct t2.a), sum(distinct t2.b), t1.c from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.c order by t1.c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | avg | sum | c -----+------------------------+-----+--------- 0 | 0.00000000000000000000 | 0 | 10value @@ -1446,6 +1527,8 @@ select distinct sum (distinct t1.a), avg(distinct t2.a), sum(distinct t2.b), t1. -- distinct on group by fields select distinct t1.c , sum(distinct t1.a), count(t2.b), sum(distinct t2.b) from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.c order by t1.c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c | sum | count | sum ---------+-----+-------+----- 10value | 0 | 8 | 0 @@ -1472,6 +1555,8 @@ select distinct t1.c , sum(distinct t1.a), count(t2.b), sum(distinct t2.b) from -- distinct on normal aggregates select distinct sum(t1.a), avg(distinct t2.a), sum(distinct (t1.a + t2.a)), t1.a, t2.b from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.a, t2.b order by t1.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | avg | sum | a | b -----+------------------------+-----+---+--- 0 | 0.00000000000000000000 | 0 | 0 | 0 @@ -1482,6 +1567,8 @@ select distinct sum(t1.a), avg(distinct t2.a), sum(distinct (t1.a + t2.a)), t1.a (5 rows) select distinct avg(t1.a + t2.b), count(distinct t1.c), count(distinct char_length(t1.c)), t1.a, t2.b from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.a, t2.b order by t1.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer avg | count | count | a | b ------------------------+-------+-------+---+--- 0.00000000000000000000 | 4 | 2 | 0 | 0 @@ -1508,6 +1595,8 @@ HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sur insert into gp_dqa_r select i , i %10, i%5 from generate_series(1,20) i; insert into gp_dqa_s select i, i %15, i%10 from generate_series(1,30) i; select a, d, count(distinct b) as c1, count(distinct c) as c2 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d, a order by a,d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer a | d | c1 | c2 ----+----+----+---- 1 | 1 | 1 | 1 @@ -1549,6 +1638,8 @@ d as c9 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c3 | c2 | c9 ----+----+----+----+---- 1 | 2 | 1 | 1 | 1 @@ -1588,6 +1679,8 @@ d as c9 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c9 ----+----+---- 1 | 1 | 1 @@ -1624,6 +1717,8 @@ select distinct count(distinct b) as c1, count(distinct c) as c2, d as c9 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c9 ----+----+---- 1 | 1 | 1 @@ -1657,6 +1752,8 @@ group by d order by c9; (28 rows) select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 from gp_dqa_r, gp_dqa_s group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer d | c1 | c2 | c9 ----+----+----+---- 1 | 10 | 5 | 1 @@ -1692,6 +1789,8 @@ select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 fro (30 rows) select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 from gp_dqa_r, gp_dqa_s group by d, a order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer d | c1 | c2 | c9 ----+----+----+---- 1 | 1 | 1 | 1 @@ -1727,18 +1826,24 @@ select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 fro (30 rows) select distinct count(distinct b) as c1, count(distinct c) as c2 from gp_dqa_r, gp_dqa_s; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 ----+---- 10 | 5 (1 row) select distinct count(distinct b) as c1, count(distinct c) as c2 from gp_dqa_r; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 ----+---- 10 | 5 (1 row) select distinct count(distinct b) as c1, count(distinct c) as c2, d, a from gp_dqa_r, gp_dqa_s where ( e = a)group by d, a order by a,d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | d | a ----+----+----+---- 1 | 1 | 1 | 1 @@ -1776,6 +1881,8 @@ ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list LINE 1: ...as c2, d from gp_dqa_r, gp_dqa_s group by d, a order by d,a; ^ select distinct count(distinct b) as c1, count(distinct c) as c2, d from gp_dqa_r, gp_dqa_s group by d, a order by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | d ----+----+---- 1 | 1 | 1 @@ -1811,6 +1918,8 @@ select distinct count(distinct b) as c1, count(distinct c) as c2, d from gp_dqa_ (30 rows) select distinct count(distinct b) as c1, count(distinct c) as c2, d from gp_dqa_r, gp_dqa_s group by d order by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | d ----+----+---- 10 | 5 | 1 @@ -1858,6 +1967,8 @@ create table gp_dqa_t2 (a int, c int) distributed by (a); insert into gp_dqa_t1 select i , i %5 from generate_series(1,10) i; insert into gp_dqa_t2 select i , i %4 from generate_series(1,10) i; select distinct A.a, sum(distinct A.b), count(distinct B.c) from gp_dqa_t1 A left join gp_dqa_t2 B on (A.a = B.a) group by A.a order by A.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer a | sum | count ----+-----+------- 1 | 1 | 1 @@ -1873,6 +1984,8 @@ select distinct A.a, sum(distinct A.b), count(distinct B.c) from gp_dqa_t1 A lef (10 rows) select distinct A.a, sum(distinct A.b), count(distinct B.c) from gp_dqa_t1 A right join gp_dqa_t2 B on (A.a = B.a) group by A.a order by A.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer a | sum | count ----+-----+------- 1 | 1 | 1 @@ -1925,12 +2038,16 @@ explain (costs off) select count(distinct d) from dqa_t1 group by i; (11 rows) select count(distinct d), count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | count -------+-------+------- 23 | 10 | 34 (1 row) select count(distinct c), count(distinct dt), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | i -------+-------+---- 5 | 9 | 3 @@ -1954,11 +2071,15 @@ create table foo_mdqa(x int, y int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Cloudberry Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. SELECT distinct C.z, count(distinct FS.x), count(distinct FS.y) FROM (SELECT 1 AS z FROM generate_series(1,10)) C, foo_mdqa FS GROUP BY z; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer z | count | count ---+-------+------- (0 rows) SELECT distinct C.z, count(distinct FS.x), count(distinct FS.y) FROM (SELECT i AS z FROM generate_series(1,10) i) C, foo_mdqa FS GROUP BY z; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer z | count | count ---+-------+------- (0 rows) @@ -1977,6 +2098,10 @@ HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sur insert into nonullstab select 1, 1 from generate_series(1, 100); -- This returns wrong result. countall(distinct a) should return 1. select countall(distinct a), count(distinct b) from nonullstab; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL countall | count ----------+------- 1 | 1 @@ -1990,12 +2115,16 @@ create table dqa_f2(x int, y int, z int) distributed by (x); insert into dqa_f1 select i%17, i%5 , i%3 from generate_series(1,1000) i; insert into dqa_f2 select i % 13, i % 5 , i % 11 from generate_series(1,1000) i; select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 136 | 10 (1 row) select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by b; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 136 | 0 @@ -2006,6 +2135,8 @@ select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0 (5 rows) select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 136 | 10 @@ -2014,12 +2145,16 @@ select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0 (3 rows) select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 78 | 10 (1 row) select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 78 | 10 @@ -2028,12 +2163,16 @@ select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), s (3 rows) select count(distinct a) filter (where a > 3),count( distinct b) filter (where a > 4), sum(distinct b) filter( where a > 4) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER count | count | sum -------+-------+----- 13 | 5 | 10 (1 row) explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------ Finalize Aggregate (cost=20.66..20.67 rows=1 width=16) @@ -2050,6 +2189,8 @@ explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (whe (11 rows) explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by b; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------ Finalize HashAggregate (cost=21.62..21.67 rows=5 width=20) @@ -2071,6 +2212,8 @@ explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (whe (16 rows) explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------- Finalize HashAggregate (cost=21.20..21.23 rows=3 width=20) @@ -2092,6 +2235,8 @@ explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (whe (16 rows) explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------ Finalize Aggregate (cost=96.41..96.42 rows=1 width=16) @@ -2114,6 +2259,8 @@ explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x (17 rows) explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Finalize HashAggregate (cost=181.11..181.14 rows=3 width=20) @@ -2141,6 +2288,8 @@ explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x (22 rows) explain select count(distinct a) filter (where a > 3),count( distinct b) filter (where a > 4), sum(distinct b) filter( where a > 4) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------ Finalize Aggregate (cost=20.67..20.68 rows=1 width=24) diff --git a/src/test/regress/sql/gp_dqa.sql b/src/test/regress/sql/gp_dqa.sql index ab8c0d91aac..808cee12c7e 100644 --- a/src/test/regress/sql/gp_dqa.sql +++ b/src/test/regress/sql/gp_dqa.sql @@ -5,6 +5,8 @@ -- the queries, so a few also use TO_CHAR() to truncate the results further. set extra_float_digits=0; +SET optimizer_trace_fallback to on; + drop table if exists dqa_t1; drop table if exists dqa_t2; From 3e053ed79f1cfcdfcf23b44c25dab70fed2ceb28 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Fri, 11 Feb 2022 12:11:26 -0800 Subject: [PATCH 12/24] Existing reloption shouldn't affect redistribution in ALTER TABLE SET DISTRIBUTE This fixes an issue where existing reloptions would cause a reorganize in situations that it should not. We used to allow ALTER TABLE SET DISTRIBUTE WITH clause to have additional storage options, and when they are present, we will re-distribute data over the segments. However since 50f2e3bbb88, we don't allow new reloptions to be supplied in the ALTER TABLE SET DISTRIBUTED WITH clause. So this commit is really a follow-up, removing redistribution decision making logic based on reloptions. Also renamed new_rel_opts() to get_rel_opts() accordingly. --- src/backend/commands/tablecmds.c | 85 +++---------------- .../expected/alter_distribution_policy.out | 50 +++++++++++ .../regress/sql/alter_distribution_policy.sql | 30 +++++++ 3 files changed, 93 insertions(+), 72 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3cc182ee5d3..982d4603aed 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -17581,8 +17581,11 @@ build_ctas_with_dist(Relation rel, DistributedBy *dist_clause, return queryDesc; } +/* + * GPDB: Convenience function to get reloptions for a given relation. + */ static Datum -new_rel_opts(Relation rel) +get_rel_opts(Relation rel) { Datum newOptions = PointerGetDatum(NULL); @@ -18127,7 +18130,6 @@ static void ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd, int numsegments) { RangeVar *tmprv; - Datum newOptions; Oid tmprelid; Oid relid = RelationGetRelid(rel); ReindexParams params = {0}; @@ -18167,10 +18169,8 @@ ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd, distby = make_distributedby_for_rel(rel); distby->numsegments = numsegments; - newOptions = new_rel_opts(rel); - queryDesc = build_ctas_with_dist(rel, distby, - untransformRelOptions(newOptions), + untransformRelOptions(get_rel_opts(rel)), &tmprv, true); @@ -18300,7 +18300,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) bool rand_pol = false; bool rep_pol = false; bool force_reorg = false; - Datum newOptions = PointerGetDatum(NULL); bool need_reorg; bool change_policy = false; int numsegments; @@ -18401,8 +18400,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) lwith = nlist; } - newOptions = new_rel_opts(rel); - if (ldistro) change_policy = true; @@ -18445,8 +18442,8 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) cmd->policy = policy; - /* only need to rebuild if have new storage options */ - if (!(DatumGetPointer(newOptions) || force_reorg)) + /* no need to rebuild if REORGANIZE=false*/ + if (!force_reorg) goto l_distro_fini; } } @@ -18465,12 +18462,9 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) policy = createReplicatedGpPolicy(ldistro->numsegments); - /* rebuild if have new storage options or policy changed */ - if (!DatumGetPointer(newOptions) && - GpPolicyIsReplicated(rel->rd_cdbpolicy)) - { + /* rebuild only if policy changed */ + if (GpPolicyIsReplicated(rel->rd_cdbpolicy)) goto l_distro_fini; - } /* * system columns is not visiable to users for replicated table, @@ -18573,11 +18567,9 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) ldistro->numsegments); /* - * See if the old policy is the same as the new one but - * remember, we still might have to rebuild if there are new - * storage options. + * See if the old policy is the same as the new one. */ - if (!DatumGetPointer(newOptions) && !force_reorg && + if (!force_reorg && (policy->nattrs == rel->rd_cdbpolicy->nattrs)) { int i; @@ -18704,7 +18696,7 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) /* Step (b) - build CTAS */ queryDesc = build_ctas_with_dist(rel, ldistro, - untransformRelOptions(newOptions), + untransformRelOptions(get_rel_opts(rel)), &tmprv, true); @@ -18796,7 +18788,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) backend_id = cmd->backendId - 1; tmprv = make_temp_table_name(rel, backend_id); - newOptions = new_rel_opts(rel); need_reorg = true; } @@ -18837,59 +18828,9 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) ReadNextMultiXactId(), NULL); - /* - * Make changes from swapping relation files visible before updating - * options below or else we get an already updated tuple error. - */ + /* Make changes from swapping relation files visible. */ CommandCounterIncrement(); - if (DatumGetPointer(newOptions)) - { - Datum repl_val[Natts_pg_class]; - bool repl_null[Natts_pg_class]; - bool repl_repl[Natts_pg_class]; - HeapTuple newOptsTuple; - HeapTuple tuple; - Relation relationRelation; - - /* - * All we need do here is update the pg_class row; the new - * options will be propagated into relcaches during - * post-commit cache inval. - */ - MemSet(repl_val, 0, sizeof(repl_val)); - MemSet(repl_null, false, sizeof(repl_null)); - MemSet(repl_repl, false, sizeof(repl_repl)); - - if (newOptions != (Datum) 0) - repl_val[Anum_pg_class_reloptions - 1] = newOptions; - else - repl_null[Anum_pg_class_reloptions - 1] = true; - - repl_repl[Anum_pg_class_reloptions - 1] = true; - - relationRelation = table_open(RelationRelationId, RowExclusiveLock); - tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(tarrelid)); - - Assert(HeapTupleIsValid(tuple)); - newOptsTuple = heap_modify_tuple(tuple, RelationGetDescr(relationRelation), - repl_val, repl_null, repl_repl); - - CatalogTupleUpdate(relationRelation, &tuple->t_self, newOptsTuple); - - heap_freetuple(newOptsTuple); - - ReleaseSysCache(tuple); - - table_close(relationRelation, RowExclusiveLock); - - /* - * Increment cmd counter to make updates visible; this is - * needed because the same tuple has to be updated again - */ - CommandCounterIncrement(); - } - /* now, reindex */ reindex_relation(tarrelid, 0, ¶ms); } diff --git a/src/test/regress/expected/alter_distribution_policy.out b/src/test/regress/expected/alter_distribution_policy.out index a5e3875ddad..1d7e50df447 100644 --- a/src/test/regress/expected/alter_distribution_policy.out +++ b/src/test/regress/expected/alter_distribution_policy.out @@ -1550,3 +1550,53 @@ select check_redistributed('insert into t_distbya select * from t_reorganize', ' reset optimizer; reset gp_force_random_redistribution; +-- When reorganize=false, we won't reorganize and this shouldn't be affected by the existing reloptions. +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) with (appendonly=false, autovacuum_enabled=false) DISTRIBUTED BY (a); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id |  +---------------+----- + 1 | 100 +(1 row) + +-- Change the distribution policy but because REORGANIZE=false, it should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id |  +---------------+----- + 1 | 100 +(1 row) + +DROP TABLE t_reorganize_false; +-- Same rule should apply to partitioned table too +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) +DISTRIBUTED BY (a) PARTITION BY RANGE(b) +( +PARTITION "00" START (0) END (1000) WITH (tablename='t_reorganize_false_0', appendonly='false', autovacuum_enabled=false), +PARTITION "01" START (1000) END (2000) WITH (tablename='t_reorganize_false_1', appendonly='false', autovacuum_enabled=false), +DEFAULT PARTITION def WITH (tablename='t_reorganize_false_def', appendonly='false', autovacuum_enabled=false) +); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id | count +---------------+------- + 1 | 100 +(1 row) + +-- Should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id | count +---------------+------- + 1 | 100 +(1 row) + +DROP TABLE t_reorganize_false; diff --git a/src/test/regress/sql/alter_distribution_policy.sql b/src/test/regress/sql/alter_distribution_policy.sql index f6479f3a974..71d9490872e 100644 --- a/src/test/regress/sql/alter_distribution_policy.sql +++ b/src/test/regress/sql/alter_distribution_policy.sql @@ -531,3 +531,33 @@ select check_redistributed('insert into t_distbya select * from t_reorganize', ' reset optimizer; reset gp_force_random_redistribution; +-- When reorganize=false, we won't reorganize and this shouldn't be affected by the existing reloptions. +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) with (appendonly=false, autovacuum_enabled=false) DISTRIBUTED BY (a); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +-- Change the distribution policy but because REORGANIZE=false, it should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +DROP TABLE t_reorganize_false; +-- Same rule should apply to partitioned table too +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) +DISTRIBUTED BY (a) PARTITION BY RANGE(b) +( +PARTITION "00" START (0) END (1000) WITH (tablename='t_reorganize_false_0', appendonly='false', autovacuum_enabled=false), +PARTITION "01" START (1000) END (2000) WITH (tablename='t_reorganize_false_1', appendonly='false', autovacuum_enabled=false), +DEFAULT PARTITION def WITH (tablename='t_reorganize_false_def', appendonly='false', autovacuum_enabled=false) +); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +-- Should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +DROP TABLE t_reorganize_false; From b12bbd950c1105b84f6b1d4b5df4c67f184419a5 Mon Sep 17 00:00:00 2001 From: Alexey Gordeev Date: Wed, 9 Jun 2021 13:47:37 +0500 Subject: [PATCH 13/24] Fix segfault on execution of multilevel correlated queries. Execution of multilevel correlated queries with high level of nesting can cause segfault(when using array_agg, json_agg) or can provide wrong results (when using classic aggs like sum()). Due to some GP limitations, correlated subqueries with skip-level correlations are not supported. Additional check condition is provided to prevent such queries from planning. QueryHasDistributedRelation function, used by this check, doesn't recurse over subplans and may return wrong results for distributed RTE_RELATION entries hided by RTE_SUBQUERY entries. Commit fixes such behavior by adding optional recursion to QueryHasDistributedRelation function. Additional regression test is included. Additional information can be found at issue #12054. --- src/backend/optimizer/plan/subselect.c | 9 +++++++-- src/include/optimizer/subselect.h | 2 +- src/test/regress/expected/qp_correlated_query.out | 6 ++++++ .../regress/expected/qp_correlated_query_optimizer.out | 6 ++++++ src/test/regress/sql/qp_correlated_query.sql | 4 ++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index ebb5bbb10a0..aaed8b99c1f 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -156,7 +156,7 @@ get_first_col_type(Plan *plan, Oid *coltype, int32 *coltypmod, /** * Returns true if query refers to a distributed table. */ -bool QueryHasDistributedRelation(Query *q) +bool QueryHasDistributedRelation(Query *q, bool recursive) { ListCell *rt = NULL; @@ -164,6 +164,11 @@ bool QueryHasDistributedRelation(Query *q) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(rt); + if (rte->rtekind == RTE_SUBQUERY + && recursive + && QueryHasDistributedRelation(rte->subquery, true)) + return true; + if (rte->relid != InvalidOid && rte->rtekind == RTE_RELATION) { @@ -325,7 +330,7 @@ make_subplan(PlannerInfo *root, Query *orig_subquery, if ((Gp_role == GP_ROLE_DISPATCH) && IsSubqueryMultiLevelCorrelated(subquery) - && QueryHasDistributedRelation(subquery)) + && QueryHasDistributedRelation(subquery, root->is_correlated_subplan)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), diff --git a/src/include/optimizer/subselect.h b/src/include/optimizer/subselect.h index 08dfb95364e..7f4c9afa950 100644 --- a/src/include/optimizer/subselect.h +++ b/src/include/optimizer/subselect.h @@ -53,6 +53,6 @@ extern bool IsSubqueryMultiLevelCorrelated(Query *sq); extern List *generate_subquery_vars(PlannerInfo *root, List *tlist, Index varno); -extern bool QueryHasDistributedRelation(Query *q); +extern bool QueryHasDistributedRelation(Query *q, bool recursive); #endif /* SUBSELECT_H */ diff --git a/src/test/regress/expected/qp_correlated_query.out b/src/test/regress/expected/qp_correlated_query.out index 07a3ab10a1c..d35e89ea6e0 100644 --- a/src/test/regress/expected/qp_correlated_query.out +++ b/src/test/regress/expected/qp_correlated_query.out @@ -1662,6 +1662,12 @@ select (select avg(x) from qp_csq_t1, qp_csq_t2 where qp_csq_t1.a = any (select 4.0000000000000000 (4 rows) +-- Planner should fail due to skip-level correlation not supported. Query should not cause segfault like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select array_agg(a_C) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported +-- Planner should fail due to skip-level correlation not supported. Query should not return wrong results like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select sum(a_C.j) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported -- ---------------------------------------------------------------------- -- Test: Correlated Subquery: CSQ with multiple columns (Heap) -- ---------------------------------------------------------------------- diff --git a/src/test/regress/expected/qp_correlated_query_optimizer.out b/src/test/regress/expected/qp_correlated_query_optimizer.out index c238b0633a4..a604629ae1e 100644 --- a/src/test/regress/expected/qp_correlated_query_optimizer.out +++ b/src/test/regress/expected/qp_correlated_query_optimizer.out @@ -1775,6 +1775,12 @@ select (select avg(x) from qp_csq_t1, qp_csq_t2 where qp_csq_t1.a = any (select 4.0000000000000000 (4 rows) +-- Planner should fail due to skip-level correlation not supported. Query should not cause segfault like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select array_agg(a_C) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported +-- Planner should fail due to skip-level correlation not supported. Query should not return wrong results like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select sum(a_C.j) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported -- ---------------------------------------------------------------------- -- Test: Correlated Subquery: CSQ with multiple columns (Heap) -- ---------------------------------------------------------------------- diff --git a/src/test/regress/sql/qp_correlated_query.sql b/src/test/regress/sql/qp_correlated_query.sql index 46048efc3b7..6c30e018391 100644 --- a/src/test/regress/sql/qp_correlated_query.sql +++ b/src/test/regress/sql/qp_correlated_query.sql @@ -342,6 +342,10 @@ SELECT a, (SELECT (SELECT d FROM qp_csq_t3 WHERE a=c)) FROM qp_csq_t1 GROUP BY a select A.i, (select C.j from C group by C.j having max(C.j) = any (select min(B.j) from B)) as C_j from A,B,C where A.i = 99 order by A.i, C_j limit 10; select (select avg(x) from qp_csq_t1, qp_csq_t2 where qp_csq_t1.a = any (select x)) as avg_x from qp_csq_t1 order by 1; +-- Planner should fail due to skip-level correlation not supported. Query should not cause segfault like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select array_agg(a_C) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +-- Planner should fail due to skip-level correlation not supported. Query should not return wrong results like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select sum(a_C.j) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; -- ---------------------------------------------------------------------- -- Test: Correlated Subquery: CSQ with multiple columns (Heap) From efae39758c3400790289b3a4c51b9d000b62f96e Mon Sep 17 00:00:00 2001 From: Jasper Li Date: Tue, 22 Feb 2022 16:34:25 +0800 Subject: [PATCH 14/24] =?UTF-8?q?set=20memory=20allocated=20by=20malloc=20?= =?UTF-8?q?to=200=20to=20fix=20'error:=20=E2=80=98data=E2=80=99=20may=20be?= =?UTF-8?q?=20used=20uninitialized'=20(#13103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * set to 0 with bzero to fix 'error: ‘data’ may be used uninitialized' * change to memset for compatibiity Co-authored-by: jasper li --- src/backend/utils/hyperloglog/gp_hyperloglog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/utils/hyperloglog/gp_hyperloglog.c b/src/backend/utils/hyperloglog/gp_hyperloglog.c index c093111fe85..f9ce8e33245 100644 --- a/src/backend/utils/hyperloglog/gp_hyperloglog.c +++ b/src/backend/utils/hyperloglog/gp_hyperloglog.c @@ -573,6 +573,7 @@ gp_hll_compress_dense(GpHLLCounter hloglog) errmsg("out of memory"), errdetail("Failed on request of size %zu.", data_rawsize))); } + memset(data, 0, data_rawsize); /* put all registers in a normal array i.e. remove dense packing so * lz compression can work optimally */ From f9a64e16068a5c055d195429862a323bd734103b Mon Sep 17 00:00:00 2001 From: "Kevin.wyh" Date: Tue, 22 Feb 2022 18:50:51 +0800 Subject: [PATCH 15/24] ORCA: avoid returning output of a CTE producer (#12776) When dealing with queries which have CTE and result set being used more than once, ORCA produces a plan with Sequence Node and ShareInputScan Node. Here is an example: explain (costs off) with tmp_t as (select id from test) select a.id from tmp_t a join tmp_t b on a.id = b.id; QUERY PLAN ------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) -> Sequence -> Shared Scan (share slice:id 1:0) -> Seq Scan on test -> Hash Join Hash Cond: (share0_ref3.id = share0_ref2.id) -> Shared Scan (share slice:id 1:0) -> Hash -> Shared Scan (share slice:id 1:0) Optimizer: Pivotal Optimizer (GPORCA) (10 rows) This plan has three ShareInputScan nodes: one producer and two consumers, all of them read from the tuplestore and return tuples to upper node. However, the CTE producer does not need to do so, because the Sequence Node only receives the last subplan's output tuples. When tuplestore is small, the extra read from the CTE producer is at low cost, but when tuplestore is huge, this read is heavy and would cause bad performance of the entire query. This commit adds a flag discard_output in ShareInputScan plan node to indicate when a ShareInputScan can avoid reading from tuplestore and return early. The flag is set to true by ORCA when translating CTE producer, and set to false elsewhere. Fixes #12710 Co-authored-by: wuyuhao28 Co-authored-by: Alexandra Wang Reviewed-by: Zhenghua Lyu Reviewed-by: Shreedhar Hardikar --- src/backend/executor/nodeShareInputScan.c | 9 +++++++++ src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp | 2 ++ src/backend/optimizer/plan/planshare.c | 1 + src/include/nodes/plannodes.h | 3 +++ 4 files changed, 15 insertions(+) diff --git a/src/backend/executor/nodeShareInputScan.c b/src/backend/executor/nodeShareInputScan.c index 1bb05edf674..4886e5d377c 100644 --- a/src/backend/executor/nodeShareInputScan.c +++ b/src/backend/executor/nodeShareInputScan.c @@ -366,6 +366,15 @@ ExecShareInputScan(PlanState *pstate) /* if first time call, need to initialize the tuplestore state. */ if (!node->isready) init_tuplestore_state(node); + + /* + * Return NULL when necessary. + * This could help improve performance, especially when tuplestore is huge, because ShareInputScan + * do not need to read tuple from tuplestore when discard_output is true, which means current + * ShareInputScan is one but not the last one of Sequence's subplans. + */ + if (sisc->discard_output) + return NULL; slot = node->ss.ps.ps_ResultTupleSlot; diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index 0a75541c1a6..65894b7d779 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -3738,6 +3738,7 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEProducerToSharedScan( // create the shared input scan representing the CTE Producer ShareInputScan *shared_input_scan = MakeNode(ShareInputScan); shared_input_scan->share_id = cte_id; + shared_input_scan->discard_output = true; Plan *plan = &(shared_input_scan->scan.plan); plan->plan_node_id = m_dxl_to_plstmt_context->GetNextPlanId(); @@ -3795,6 +3796,7 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEConsumerToSharedScan( ShareInputScan *share_input_scan_cte_consumer = MakeNode(ShareInputScan); share_input_scan_cte_consumer->share_id = cte_id; + share_input_scan_cte_consumer->discard_output = false; Plan *plan = &(share_input_scan_cte_consumer->scan.plan); plan->plan_node_id = m_dxl_to_plstmt_context->GetNextPlanId(); diff --git a/src/backend/optimizer/plan/planshare.c b/src/backend/optimizer/plan/planshare.c index 1b5c53f549e..75983b8ffe3 100644 --- a/src/backend/optimizer/plan/planshare.c +++ b/src/backend/optimizer/plan/planshare.c @@ -39,6 +39,7 @@ make_shareinputscan(PlannerInfo *root, Plan *inputplan) sisc->producer_slice_id = -1; sisc->this_slice_id = -1; sisc->nconsumers = 0; + sisc->discard_output = false; sisc->scan.plan.qual = NIL; sisc->scan.plan.righttree = NULL; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 64679cbe527..fbac2ca02b9 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -1099,6 +1099,9 @@ typedef struct ShareInputScan /* Number of consumer slices participating, not including the producer. */ int nconsumers; + + /* Discard the scan output? True for ORCA CTE producer, false otherwise. */ + bool discard_output; } ShareInputScan; /* ---------------- From 6f5c90ce91e80d20266e71a8c09c8ac4ba5b0af6 Mon Sep 17 00:00:00 2001 From: Alexey Gordeev <42867699+InnerLife0@users.noreply.github.com> Date: Wed, 23 Feb 2022 10:51:55 +0500 Subject: [PATCH 16/24] Fix index corruption when invalid snapshot used with AO tables. (#12863) Using of regular MVCC snapshot to access AO-tables metadata may cause index building (performed globally) inconsistency, because tuples inserted (the same globally) at another opened transactions are not visible. Commit fixes such behavior using SnapshotSelf for metadata access. Additional isolation test shows how to reproduce the bug before the fix. --- src/backend/access/aocs/aocsam.c | 6 +- src/backend/access/appendonly/appendonlyam.c | 6 +- .../uao/snapshot_index_corruption.source | 35 ++++++++ src/test/isolation2/isolation2_schedule | 2 + .../uao/snapshot_index_corruption.source | 83 +++++++++++++++++++ 5 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 src/test/isolation2/input/uao/snapshot_index_corruption.source create mode 100644 src/test/isolation2/output/uao/snapshot_index_corruption.source diff --git a/src/backend/access/aocs/aocsam.c b/src/backend/access/aocs/aocsam.c index 0ae994659b1..bbc42dc81c3 100644 --- a/src/backend/access/aocs/aocsam.c +++ b/src/backend/access/aocs/aocsam.c @@ -507,13 +507,15 @@ aocs_beginscan(Relation relation, RelationIncrementReferenceCount(relation); /* - * the append-only meta data should never be fetched with + * The append-only meta data should never be fetched with * SnapshotAny as bogus results are returned. + * We use SnapshotSelf for metadata, as regular MVCC snapshot can hide newly + * globally inserted tuples from global index build process. */ if (snapshot != SnapshotAny) aocsMetaDataSnapshot = snapshot; else - aocsMetaDataSnapshot = GetTransactionSnapshot(); + aocsMetaDataSnapshot = SnapshotSelf; seginfo = GetAllAOCSFileSegInfo(relation, aocsMetaDataSnapshot, &total_seg, NULL); return aocs_beginscan_internal(relation, diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c index d4681bd4aa3..32a306f3ae0 100755 --- a/src/backend/access/appendonly/appendonlyam.c +++ b/src/backend/access/appendonly/appendonlyam.c @@ -1638,10 +1638,12 @@ appendonly_beginscan(Relation relation, if (appendOnlyMetaDataSnapshot == SnapshotAny) { /* - * the append-only meta data should never be fetched with + * The append-only meta data should never be fetched with * SnapshotAny as bogus results are returned. + * We use SnapshotSelf for metadata, as regular MVCC snapshot can hide + * newly globally inserted tuples from global index build process. */ - appendOnlyMetaDataSnapshot = GetTransactionSnapshot(); + appendOnlyMetaDataSnapshot = SnapshotSelf; } /* diff --git a/src/test/isolation2/input/uao/snapshot_index_corruption.source b/src/test/isolation2/input/uao/snapshot_index_corruption.source new file mode 100644 index 00000000000..48c989b1af7 --- /dev/null +++ b/src/test/isolation2/input/uao/snapshot_index_corruption.source @@ -0,0 +1,35 @@ +-- @Description Test index corruption when invalid snapshot used. +-- +-- Create AO table, insert few rows on it. +drop table if exists test_ao; +create table test_ao(i bigint) using @amname@ distributed by (i); +insert into test_ao select generate_series(1,100); +-- Test 1 +-- Begin single-insert transaction. +1: begin; +1: insert into test_ao values(101); +-- Try to create index, it should hold on lock before commit below. +2&: create index test_ao_idx on test_ao(i); +-- Commit single-insert transaction, so index continues creation. +1: commit; +-- Force index usage and check row is here (false before fix). +2<: +2: set optimizer=off; +2: set enable_seqscan=off; +2: explain (costs off) select i from test_ao where i = 101; +2: select i from test_ao where i = 101; + +-- Test 2 +-- Drop incomplete index +1: drop index test_ao_idx; +-- Check row is here and start repeatable read transaction. +2: select i from test_ao where i = 100; +2: begin; +2: set transaction isolation level repeatable read; +2: select 1; +-- Update row selected above and create new index +1: update test_ao set i = 200 where i = 100; +1: create index test_ao_idx on test_ao(i); +-- For the repeatable read isolation level row still there. +2: explain (costs off) select i from test_ao where i = 100; +2: select i from test_ao where i = 100; diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 806c62a4d7f..03f9f7c467d 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -144,6 +144,7 @@ test: uao/select_while_vacuum_serializable2_row test: uao/selectinsert_while_vacuum_row test: uao/selectinsertupdate_while_vacuum_row test: uao/selectupdate_while_vacuum_row +test: uao/snapshot_index_corruption_row test: uao/update_while_vacuum_row test: uao/vacuum_self_serializable_row test: uao/vacuum_self_serializable2_row @@ -200,6 +201,7 @@ test: uao/select_while_vacuum_serializable2_column test: uao/selectinsert_while_vacuum_column test: uao/selectinsertupdate_while_vacuum_column test: uao/selectupdate_while_vacuum_column +test: uao/snapshot_index_corruption_column test: uao/update_while_vacuum_column test: uao/vacuum_self_serializable_column test: uao/vacuum_self_serializable2_column diff --git a/src/test/isolation2/output/uao/snapshot_index_corruption.source b/src/test/isolation2/output/uao/snapshot_index_corruption.source new file mode 100644 index 00000000000..59b00e9f10c --- /dev/null +++ b/src/test/isolation2/output/uao/snapshot_index_corruption.source @@ -0,0 +1,83 @@ +-- @Description Test index corruption when invalid snapshot used. +-- +-- Create AO table, insert few rows on it. +drop table if exists test_ao; +DROP +create table test_ao(i bigint) using @amname@ distributed by (i); +CREATE +insert into test_ao select generate_series(1,100); +INSERT 100 +-- Test 1 +-- Begin single-insert transaction. +1: begin; +BEGIN +1: insert into test_ao values(101); +INSERT 1 +-- Try to create index, it should hold on lock before commit below. +2&: create index test_ao_idx on test_ao(i); +-- Commit single-insert transaction, so index continues creation. +1: commit; +COMMIT +-- Force index usage and check row is here (false before fix). +2<: <... completed> +CREATE +2: set optimizer=off; +SET +2: set enable_seqscan=off; +SET +2: explain (costs off) select i from test_ao where i = 101; + QUERY PLAN +---------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) + -> Bitmap Heap Scan on test_ao + Recheck Cond: (i = 101) + -> Bitmap Index Scan on test_ao_idx + Index Cond: (i = 101) + Optimizer: Postgres query optimizer +(6 rows) +2: select i from test_ao where i = 101; + i +----- + 101 +(1 row) + +-- Test 2 +-- Drop incomplete index +1: drop index test_ao_idx; +DROP +-- Check row is here and start repeatable read transaction. +2: select i from test_ao where i = 100; + i +----- + 100 +(1 row) +2: begin; +BEGIN +2: set transaction isolation level repeatable read; +SET +2: select 1; + ?column? +---------- + 1 +(1 row) +-- Update row selected above and create new index +1: update test_ao set i = 200 where i = 100; +UPDATE 1 +1: create index test_ao_idx on test_ao(i); +CREATE +-- For the repeatable read isolation level row still there. +2: explain (costs off) select i from test_ao where i = 100; + QUERY PLAN +---------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) + -> Bitmap Heap Scan on test_ao + Recheck Cond: (i = 100) + -> Bitmap Index Scan on test_ao_idx + Index Cond: (i = 100) + Optimizer: Postgres query optimizer +(6 rows) +2: select i from test_ao where i = 100; + i +----- + 100 +(1 row) From 3207762684f548c83e3ec7f64539108c26bd8e1c Mon Sep 17 00:00:00 2001 From: xiaoxiao <53000479+xiaoxiaoHe-E@users.noreply.github.com> Date: Thu, 24 Feb 2022 10:44:44 +0800 Subject: [PATCH 17/24] et staging table schema to gpload:external:schema in yaml file (#13110) --- gpMgmt/bin/gpload.py | 24 ++++++++++++++-------- gpMgmt/bin/gpload_test/gpload2/query33.ans | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index a6ce8f471dc..2d7c0a1eb7f 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -2254,9 +2254,9 @@ def get_reuse_staging_table_query(self, encoding_conditions): return: sql(string) ''' - sql = """SELECT oid::regclass - FROM pg_class - WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions) + sql = """SELECT oid::regclass \ +FROM pg_class \ +WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions) self.log(self.DEBUG, "query used to identify reusable temporary relations: %s" % sql) return sql @@ -2273,7 +2273,8 @@ def get_table_oid(self, tableName): pass return None - def get_ext_schematable(self, schemaName, tableName): + + def get_schematable(self, schemaName, tableName): ''' return formated table name ''' @@ -2448,7 +2449,7 @@ def create_external_table(self): AND n.nspname !~ '^pg_toast'""" result = self.db.query(sql).getresult() if len(result) > 0: - self.extSchemaTable = self.get_ext_schematable(quote_unident(self.extSchemaName), self.extTableName) + self.extSchemaTable = self.get_schematable(quote_unident(self.extSchemaName), self.extTableName) self.log(self.INFO, "reusing external staging table %s" % self.extSchemaTable) return # staging table is not specified, we need to find it manually @@ -2468,7 +2469,7 @@ def create_external_table(self): self.extTableName = (resultList[0])[0] # fast match result is only table name, so we need add schema info if self.fast_match: - self.extSchemaTable = self.get_ext_schematable(quote_unident(self.extSchemaName), self.extTableName) + self.extSchemaTable = self.get_schematable(quote_unident(self.extSchemaName), self.extTableName) else: self.extSchemaTable = self.extTableName self.log(self.INFO, "reusing external table %s" % self.extSchemaTable) @@ -2479,13 +2480,13 @@ def create_external_table(self): # around self.extTableName = "ext_gpload_reusable_%s" % self.unique_suffix - self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.get_ext_schematable(self.extSchemaName, self.extTableName)) + self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.get_schematable(self.extSchemaName, self.extTableName)) # process the single quotes in order to successfully create an external table. self.formatOpts = self.formatOpts.replace("'\''","E'\\''") # construct a CREATE EXTERNAL TABLE statement and execute it - self.extSchemaTable = self.get_ext_schematable(self.extSchemaName, self.extTableName) + self.extSchemaTable = self.get_schematable(self.extSchemaName, self.extTableName) sql = "create external table %s" % self.extSchemaTable sql += "(%s)" % ','.join(['%s %s' % (a[0], a[1]) for a in from_cols]) @@ -2566,10 +2567,12 @@ def create_staging_table(self): # we no longer need the timestamp, since we will never want to create few # tables with same encoding_conditions self.staging_table_name = "staging_gpload_reusable_%s" % (encoding_conditions) + self.staging_table_name = self.get_schematable(self.extSchemaName, self.staging_table_name) self.log(self.INFO, "did not find a staging table to reuse. creating %s" % self.staging_table_name) # MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table, # and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True. + sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name) cols = ['"%s" %s' % (a[0], a[1]) for a in target_columns] sql += "(%s)" % ','.join(cols) @@ -2882,6 +2885,7 @@ def do_method(self): # Is the table to be truncated before the load? preload = self.getconfig('gpload:preload', list, default=None) method = self.getconfig('gpload:output:mode', str, 'insert').lower() + external = self.getconfig('gpload:external', list, default=None) self.log_errors = self.getconfig('gpload:input:log_errors', bool, False) truncate = False self.reuse_tables = False @@ -2889,6 +2893,10 @@ def do_method(self): if not self.options.no_auto_trans and not method=='insert': self.db.query("BEGIN") + self.extSchemaName = self.getconfig('gpload:external:schema', str, None) + if self.extSchemaName == '%': + self.extSchemaName = self.schema + if preload: truncate = self.getconfig('gpload:preload:truncate',bool,False) self.reuse_tables = self.getconfig('gpload:preload:reuse_tables',bool,False) diff --git a/gpMgmt/bin/gpload_test/gpload2/query33.ans b/gpMgmt/bin/gpload_test/gpload2/query33.ans index b69631bff50..93756fe0acf 100644 --- a/gpMgmt/bin/gpload_test/gpload2/query33.ans +++ b/gpMgmt/bin/gpload_test/gpload2/query33.ans @@ -1,7 +1,7 @@ 2018-07-24 06:14:29|INFO|gpload session started 2018-07-24 06:14:29 2018-07-24 06:14:29|INFO|setting schema 'public' for table 'texttable' 2018-07-24 06:14:29|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2018-07-24 06:14:29|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 +2018-07-24 06:14:29|INFO|did not find a staging table to reuse. creating test.staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 2018-07-24 06:14:29|INFO|did not find an external table to reuse. creating test.ext_gpload_reusable_d2e95f76_8f08_11e8_8c76_0242ac110002 2018-07-24 06:14:29|INFO|running time: 0.40 seconds 2018-07-24 06:14:29|INFO|rows Inserted = 16 @@ -11,7 +11,7 @@ 2018-07-24 06:14:30|INFO|gpload session started 2018-07-24 06:14:30 2018-07-24 06:14:30|INFO|setting schema 'public' for table 'texttable' 2018-07-24 06:14:30|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2018-07-24 06:14:30|INFO|reusing staging table staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 +2018-07-24 06:14:30|INFO|reusing staging table test.staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 2018-07-24 06:14:30|INFO|reusing external table test.ext_gpload_reusable_d2e95f76_8f08_11e8_8c76_0242ac110002 2018-07-24 06:14:30|INFO|running time: 0.31 seconds 2018-07-24 06:14:30|INFO|rows Inserted = 0 From 5d1134a43f8e608e4c6b98c2c1522fb129113f52 Mon Sep 17 00:00:00 2001 From: Xing Guo Date: Fri, 25 Feb 2022 09:29:34 +0800 Subject: [PATCH 18/24] Fix incorrect amount of memory allocated for WindowAgg. (#13124) This patch helps fix incorrect amount of memory allocated for WindowAgg. Greenplum uses statement_mem rather than work_mem to control the memory allocated to queries. Besides, test cases are attached for this patch. --- src/backend/executor/nodeWindowAgg.c | 54 +++++- src/test/regress/expected/misc_jiras.out | 34 +++- .../expected/statement_mem_for_windowagg.out | 176 ++++++++++++++++++ src/test/regress/greenplum_schedule | 4 +- src/test/regress/sql/misc_jiras.sql | 16 +- .../sql/statement_mem_for_windowagg.sql | 57 ++++++ 6 files changed, 328 insertions(+), 13 deletions(-) create mode 100644 src/test/regress/expected/statement_mem_for_windowagg.out create mode 100644 src/test/regress/sql/statement_mem_for_windowagg.sql diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index 75933ca5488..6a626f1de77 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -40,6 +40,7 @@ #include "executor/executor.h" #include "executor/nodeWindowAgg.h" #include "miscadmin.h" +#include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parse_agg.h" @@ -49,6 +50,7 @@ #include "utils/builtins.h" #include "utils/datum.h" #include "utils/expandeddatum.h" +#include "utils/faultinjector.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/regproc.h" @@ -269,7 +271,7 @@ initialize_windowaggregate(WindowAggState *winstate, peraggstate->distinctLtOper, peraggstate->distinctColl, false, /* nullsFirstFlag */ - work_mem, + PlanStateOperatorMemKB((PlanState *) winstate), NULL, /* coordinate */ false); } @@ -686,6 +688,24 @@ perform_distinct_windowaggregate(WindowAggState *winstate, tuplesort_performsort(peraggstate->distinctSortState); +#ifdef FAULT_INJECTOR + /* + * This routine is used for tracing whether the sort operation of DISTINCT-qualified + * WindowAgg spills to disk. + */ + if (SIMPLE_FAULT_INJECTOR("distinct_winagg_perform_sort") == FaultInjectorTypeSkip) + { + TuplesortInstrumentation sortstats; + tuplesort_get_stats(peraggstate->distinctSortState, &sortstats); + if (sortstats.spaceType == SORT_SPACE_TYPE_MEMORY) + ereport(NOTICE, + (errmsg("distinct winagg sortstats: sort operation fitted in memory"))); + else + ereport(NOTICE, + (errmsg("distinct winagg sortstats: sort operation spilled to disk"))); + } +#endif + /* load the first tuple from spool */ if (tuplesort_getdatum(peraggstate->distinctSortState, true, &fcinfo->args[1].value, &fcinfo->args[1].isnull, NULL)) @@ -1360,7 +1380,9 @@ begin_partition(WindowAggState *winstate) } /* Create new tuplestore for this partition */ - winstate->buffer = tuplestore_begin_heap(false, false, work_mem); + winstate->buffer = + tuplestore_begin_heap(false, false, + PlanStateOperatorMemKB((PlanState *) winstate)); /* * Set up read pointers for the tuplestore. The current pointer doesn't @@ -2426,6 +2448,34 @@ ExecWindowAgg(PlanState *pstate) */ spool_tuples(winstate, winstate->currentpos); +#ifdef FAULT_INJECTOR + /* + * This routine is used for testing if we have allocated enough memory + * for the tuplestore (winstate->buffer) in begin_partition(). If all + * tuples of the current partition can be fitted in the memory, we + * emit a notice saying 'fitted in memory'. If they cannot be fitted in + * the memory, we emit a notice saying 'spilled to disk'. If there're + * no input rows, we emit a notice saying 'no input rows'. + * + * NOTE: The fault-injector only triggers once, we emit the notice when + * we finishes spooling all the tuples of the first partition. + */ + if (winstate->partition_spooled && + winstate->currentpos >= winstate->spooled_rows && + SIMPLE_FAULT_INJECTOR("winagg_after_spool_tuples") == FaultInjectorTypeSkip) + { + if (winstate->buffer) + { + if (tuplestore_in_memory(winstate->buffer)) + ereport(NOTICE, (errmsg("winagg: tuplestore fitted in memory"))); + else + ereport(NOTICE, (errmsg("winagg: tuplestore spilled to disk"))); + } + else + ereport(NOTICE, (errmsg("winagg: no input rows"))); + } +#endif + /* Move to the next partition if we reached the end of this partition */ if (winstate->partition_spooled && winstate->currentpos >= winstate->spooled_rows) diff --git a/src/test/regress/expected/misc_jiras.out b/src/test/regress/expected/misc_jiras.out index 2c9e78fab5a..ebe7446972e 100644 --- a/src/test/regress/expected/misc_jiras.out +++ b/src/test/regress/expected/misc_jiras.out @@ -1,4 +1,5 @@ drop schema if exists misc_jiras; +NOTICE: schema "misc_jiras" does not exist, skipping create schema misc_jiras; -- -- Test backward scanning of tuplestore spill files. @@ -12,11 +13,21 @@ create schema misc_jiras; create table misc_jiras.t1 (c1 int, c2 text, c3 smallint) distributed by (c1); insert into misc_jiras.t1 select i % 13, md5(i::text), i % 3 from generate_series(1, 20000) i; --- tuplestore uses work_mem to control the in-memory data size, set a small --- value to trigger the spilling. -set work_mem to '64kB'; -WARNING: "work_mem": setting is deprecated, and may be removed in a future release. +-- tuplestore in windowagg uses statement_mem to control the in-memory data size, +-- set a small value to trigger the spilling. +set statement_mem to '512kB'; set extra_float_digits=0; -- the last decimal digits are somewhat random +-- Inject fault at 'winagg_after_spool_tuples' to show that the tuplestore spills +-- to disk. +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + select sum(cc) from ( select c1 , c2 @@ -28,13 +39,24 @@ select sum(cc) from ( from misc_jiras.t1 group by 1, 2 ) tt; +NOTICE: winagg: tuplestore spilled to disk (seg0 slice1 127.0.0.1:7002 pid=54719) +NOTICE: winagg: tuplestore spilled to disk (seg1 slice1 127.0.0.1:7003 pid=54720) +NOTICE: winagg: tuplestore spilled to disk (seg2 slice1 127.0.0.1:7004 pid=54721) sum --------- 10006.5 (1 row) -reset work_mem; -WARNING: "work_mem": setting is deprecated, and may be removed in a future release. +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +reset statement_mem; -- non-ASCII multibyte character should show up correctly in error messages. select '溋' || (B'1'); ERROR: "溋" is not a valid binary digit diff --git a/src/test/regress/expected/statement_mem_for_windowagg.out b/src/test/regress/expected/statement_mem_for_windowagg.out new file mode 100644 index 00000000000..a2e44474ec3 --- /dev/null +++ b/src/test/regress/expected/statement_mem_for_windowagg.out @@ -0,0 +1,176 @@ +CREATE TABLE dummy_table(x int, y int) DISTRIBUTED BY (y); +INSERT INTO dummy_table SELECT generate_series(0, 10000), 0; +INSERT INTO dummy_table SELECT generate_series(0, 10000), 3; +INSERT INTO dummy_table SELECT generate_series(0, 10000), 10; +-- 1. Test that if we set statement_mem to a larger value, the tuplestore +-- for caching the tuples in partition used in WindowAgg is able to be fitted +-- in memory. +SET statement_mem TO '2048kB'; +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: winagg: tuplestore fitted in memory (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: winagg: tuplestore fitted in memory (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: winagg: tuplestore fitted in memory (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=6.520..15.872 rows=30003 loops=1) + -> WindowAgg (cost=0.00..431.00 rows=1 width=8) (actual time=10.607..13.115 rows=10001 loops=1) + Partition By: y + -> Sort (cost=0.00..431.00 rows=1 width=8) (actual time=5.298..6.324 rows=10001 loops=1) + Sort Key: y + Sort Method: quicksort Memory: 5040kB + -> Seq Scan on dummy_table (cost=0.00..431.00 rows=1 width=8) (actual time=0.036..3.299 rows=10001 loops=1) + Planning Time: 5.241 ms + (slice0) Executor memory: 48K bytes. + (slice1) Executor memory: 606K bytes avg x 3 workers, 606K bytes max (seg0). Work_mem: 606K bytes max. + Memory used: 2048kB + Optimizer: Pivotal Optimizer (GPORCA) + Execution Time: 17.225 ms +(13 rows) + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- 2. Test that if we set statement_mem to a smaller value, the tuplestore +-- for caching the tuples in partition used in WindowAgg will be spilled to disk. +SET statement_mem TO '1024kB'; +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: winagg: tuplestore spilled to disk (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: winagg: tuplestore spilled to disk (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: winagg: tuplestore spilled to disk (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=8.784..13.923 rows=30003 loops=1) + -> WindowAgg (cost=0.00..431.00 rows=1 width=8) (actual time=8.390..9.720 rows=10001 loops=1) + Partition By: y + -> Sort (cost=0.00..431.00 rows=1 width=8) (actual time=3.125..4.135 rows=10001 loops=1) + Sort Key: y + Sort Method: external merge Disk: 6144kB + -> Seq Scan on dummy_table (cost=0.00..431.00 rows=1 width=8) (actual time=0.032..1.589 rows=10001 loops=1) + Planning Time: 3.174 ms + (slice0) Executor memory: 42K bytes. + (slice1) Executor memory: 391K bytes avg x 3 workers, 391K bytes max (seg0). Work_mem: 391K bytes max. + Memory used: 1024kB + Optimizer: Pivotal Optimizer (GPORCA) + Execution Time: 15.235 ms +(13 rows) + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- 3. Test that if we set statement_mem to a larger value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg is able to be fitted in memory. +SET statement_mem TO '1024kB'; +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: distinct winagg sortstats: sort operation fitted in memory (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: distinct winagg sortstats: sort operation fitted in memory (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: distinct winagg sortstats: sort operation fitted in memory (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2446.06..4096.31 rows=86100 width=36) (actual time=10.348..16.027 rows=30003 loops=1) + -> WindowAgg (cost=2446.06..2948.31 rows=28700 width=36) (actual time=10.186..11.795 rows=10001 loops=1) + Partition By: y + -> Sort (cost=2446.06..2517.81 rows=28700 width=8) (actual time=3.678..4.683 rows=10001 loops=1) + Sort Key: y + Sort Method: external merge Disk: 6144kB + -> Seq Scan on dummy_table (cost=0.00..321.00 rows=28700 width=8) (actual time=0.030..1.919 rows=10001 loops=1) + Planning Time: 1.695 ms + (slice0) Executor memory: 50K bytes. + (slice1) Executor memory: 391K bytes avg x 3 workers, 391K bytes max (seg0). Work_mem: 391K bytes max. + Memory used: 1024kB + Optimizer: Postgres query optimizer + Execution Time: 18.747 ms +(13 rows) + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +-- 4. Test that if we set statement_mem to a smaller value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg will be spilled to disk. +SET statement_mem TO '128kB'; +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: distinct winagg sortstats: sort operation spilled to disk (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: distinct winagg sortstats: sort operation spilled to disk (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: distinct winagg sortstats: sort operation spilled to disk (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2446.06..4096.31 rows=86100 width=36) (actual time=13.040..19.232 rows=30003 loops=1) + -> WindowAgg (cost=2446.06..2948.31 rows=28700 width=36) (actual time=12.768..14.527 rows=10001 loops=1) + Partition By: y + -> Sort (cost=2446.06..2517.81 rows=28700 width=8) (actual time=4.278..5.449 rows=10001 loops=1) + Sort Key: y + Sort Method: external merge Disk: 9216kB + -> Seq Scan on dummy_table (cost=0.00..321.00 rows=28700 width=8) (actual time=0.029..1.746 rows=10001 loops=1) + Planning Time: 1.509 ms + (slice0) Executor memory: 39K bytes. + (slice1) Executor memory: 275K bytes avg x 3 workers, 275K bytes max (seg0). Work_mem: 275K bytes max. + Memory used: 128kB + Optimizer: Postgres query optimizer + Execution Time: 22.056 ms +(13 rows) + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +-- Do some clean-ups. +DROP TABLE dummy_table; +RESET statement_mem; diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index c12d6ad33f5..66c5129c356 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -54,10 +54,12 @@ test: leastsquares opr_sanity_gp decode_expr bitmapscan bitmapscan_ao case_gp li test: gpcopy test: orca_static_pruning orca_groupingsets_fallbacks -test: filter gpctas gpdist gpdist_opclasses gpdist_legacy_opclasses matrix sublink table_functions olap_setup complex opclass_ddl information_schema guc_env_var gp_explain distributed_transactions explain_format olap_plans misc_jiras gp_copy_dtx +test: filter gpctas gpdist gpdist_opclasses gpdist_legacy_opclasses matrix sublink table_functions olap_setup complex opclass_ddl information_schema guc_env_var gp_explain distributed_transactions explain_format olap_plans gp_copy_dtx # below test(s) inject faults so each of them need to be in a separate group test: guc_gp test: toast +test: misc_jiras +test: statement_mem_for_windowagg # namespace_gp test will show diff if concurrent tests use temporary tables. # So run it separately. diff --git a/src/test/regress/sql/misc_jiras.sql b/src/test/regress/sql/misc_jiras.sql index b90e908f37c..1536496d505 100644 --- a/src/test/regress/sql/misc_jiras.sql +++ b/src/test/regress/sql/misc_jiras.sql @@ -15,12 +15,17 @@ create table misc_jiras.t1 (c1 int, c2 text, c3 smallint) distributed by (c1); insert into misc_jiras.t1 select i % 13, md5(i::text), i % 3 from generate_series(1, 20000) i; --- tuplestore uses work_mem to control the in-memory data size, set a small --- value to trigger the spilling. -set work_mem to '64kB'; +-- tuplestore in windowagg uses statement_mem to control the in-memory data size, +-- set a small value to trigger the spilling. +set statement_mem to '512kB'; set extra_float_digits=0; -- the last decimal digits are somewhat random +-- Inject fault at 'winagg_after_spool_tuples' to show that the tuplestore spills +-- to disk. +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + select sum(cc) from ( select c1 , c2 @@ -33,7 +38,10 @@ select sum(cc) from ( group by 1, 2 ) tt; -reset work_mem; +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +reset statement_mem; -- non-ASCII multibyte character should show up correctly in error messages. select '溋' || (B'1'); diff --git a/src/test/regress/sql/statement_mem_for_windowagg.sql b/src/test/regress/sql/statement_mem_for_windowagg.sql new file mode 100644 index 00000000000..28339d55ba7 --- /dev/null +++ b/src/test/regress/sql/statement_mem_for_windowagg.sql @@ -0,0 +1,57 @@ +CREATE TABLE dummy_table(x int, y int) DISTRIBUTED BY (y); +INSERT INTO dummy_table SELECT generate_series(0, 10000), 0; +INSERT INTO dummy_table SELECT generate_series(0, 10000), 3; +INSERT INTO dummy_table SELECT generate_series(0, 10000), 10; + +-- 1. Test that if we set statement_mem to a larger value, the tuplestore +-- for caching the tuples in partition used in WindowAgg is able to be fitted +-- in memory. +SET statement_mem TO '2048kB'; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- 2. Test that if we set statement_mem to a smaller value, the tuplestore +-- for caching the tuples in partition used in WindowAgg will be spilled to disk. +SET statement_mem TO '1024kB'; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- 3. Test that if we set statement_mem to a larger value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg is able to be fitted in memory. +SET statement_mem TO '1024kB'; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- 4. Test that if we set statement_mem to a smaller value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg will be spilled to disk. +SET statement_mem TO '128kB'; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- Do some clean-ups. +DROP TABLE dummy_table; +RESET statement_mem; From 4f6d7920c014ef7908b1c025c515d69fb892292e Mon Sep 17 00:00:00 2001 From: Adam Lee Date: Thu, 24 Feb 2022 06:59:28 +0000 Subject: [PATCH 19/24] Remove num_segments option from the foreign table layer aa7c74cfa47 - Support num_segments option for foreign servers and tables Commit above introduces the support of num_segments option, however in practice the option for foreign tables confuses the user. This commit removes it from the foreign table layer, only the foreign servers support. --- src/backend/commands/foreigncmds.c | 1 - src/backend/foreign/foreign.c | 9 +-------- src/backend/optimizer/util/pathnode.c | 14 +++++++++++-- src/backend/optimizer/util/plancat.c | 3 --- src/include/foreign/foreign.h | 3 +-- src/include/nodes/pathnodes.h | 3 +-- src/test/regress/expected/gp_foreign_data.out | 20 ++++++++----------- src/test/regress/sql/gp_foreign_data.sql | 19 +++++++++--------- 8 files changed, 33 insertions(+), 39 deletions(-) diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index b39b2c69b43..a61b0b71d35 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -191,7 +191,6 @@ transformGenericOptions(Oid catalogId, if (catalogId != UserMappingRelationId) { SeparateOutMppExecute(&resultOptions); - SeparateOutNumSegments(&resultOptions); } if (OidIsValid(fdwvalidator)) diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index ed619d47348..4e55bf5d606 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -366,20 +366,13 @@ GetForeignTable(Oid relid) else ft->options = untransformRelOptions(datum); - ForeignServer *server = GetForeignServer(ft->serverid); - ft->exec_location = SeparateOutMppExecute(&ft->options); if (ft->exec_location == FTEXECLOCATION_NOT_DEFINED) { + ForeignServer *server = GetForeignServer(ft->serverid); ft->exec_location = server->exec_location; } - ft->num_segments = SeparateOutNumSegments(&ft->options); - if (ft->num_segments <= 0) - { - ft->num_segments = server->num_segments; - } - ReleaseSysCache(tp); return ft; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 71b8ba267e1..540d27abc48 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -3639,13 +3639,18 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->path.total_cost = total_cost; pathnode->path.pathkeys = pathkeys; + ForeignServer *server = NULL; switch (rel->exec_location) { case FTEXECLOCATION_ANY: CdbPathLocus_MakeGeneral(&(pathnode->path.locus)); break; case FTEXECLOCATION_ALL_SEGMENTS: - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), rel->num_segments, 0); + server = GetForeignServer(rel->serverid); + if (server) + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments); + else + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount()); break; case FTEXECLOCATION_COORDINATOR: CdbPathLocus_MakeEntry(&(pathnode->path.locus)); @@ -3704,13 +3709,18 @@ create_foreign_join_path(PlannerInfo *root, RelOptInfo *rel, pathnode->path.total_cost = total_cost; pathnode->path.pathkeys = pathkeys; + ForeignServer *server = NULL; switch (rel->exec_location) { case FTEXECLOCATION_ANY: CdbPathLocus_MakeGeneral(&(pathnode->path.locus)); break; case FTEXECLOCATION_ALL_SEGMENTS: - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), rel->num_segments, 0); + server = GetForeignServer(rel->serverid); + if (server) + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments); + else + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount()); break; case FTEXECLOCATION_COORDINATOR: CdbPathLocus_MakeEntry(&(pathnode->path.locus)); diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 1e64cfb06ee..95662fd05f8 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -60,7 +60,6 @@ #include "cdb/cdbappendonlyam.h" #include "cdb/cdbrelsize.h" -#include "cdb/cdbutil.h" #include "catalog/pg_appendonly.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_inherits.h" @@ -472,14 +471,12 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, rel->serverid = GetForeignServerIdByRelId(RelationGetRelid(relation)); rel->fdwroutine = GetFdwRoutineForRelation(relation, true); rel->exec_location = GetForeignTable(RelationGetRelid(relation))->exec_location; - rel->num_segments = GetForeignTable(RelationGetRelid(relation))->num_segments; } else { rel->serverid = InvalidOid; rel->fdwroutine = NULL; rel->exec_location = FTEXECLOCATION_NOT_DEFINED; - rel->num_segments = getgpsegmentCount(); } /* Collect info about relation's foreign keys, if relevant */ diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index c8226060067..04405241b0c 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -59,8 +59,7 @@ typedef struct ForeignTable Oid relid; /* relation Oid */ Oid serverid; /* server Oid */ List *options; /* ftoptions as DefElem list */ - char exec_location; /* execute on COORDINATOR, ANY or ALL SEGMENTS, Cloudberry MPP specific */ - int32 num_segments; /* the number of segments of the foreign table */ + char exec_location; /* execute on COORDINATOR, ANY or ALL SEGMENTS, Greenplum MPP specific */ } ForeignTable; /* Flags for GetForeignServerExtended */ diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 695b4dbc5e3..6cb36977a2b 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -924,8 +924,7 @@ typedef struct RelOptInfo Oid serverid; /* identifies server for the table or join */ Oid userid; /* identifies user to check access as */ bool useridiscurrent; /* join is only valid for current user */ - char exec_location; /* execute on MASTER, ANY or ALL SEGMENTS, Cloudberry MPP specific */ - int32 num_segments; /* number of segments, Cloudberry MPP specific */ + char exec_location; /* execute on MASTER, ANY or ALL SEGMENTS, Greenplum MPP specific */ /* use "struct FdwRoutine" to avoid including fdwapi.h here */ struct FdwRoutine *fdwroutine; void *fdw_private; diff --git a/src/test/regress/expected/gp_foreign_data.out b/src/test/regress/expected/gp_foreign_data.out index c4dbd5bea93..500d395404d 100644 --- a/src/test/regress/expected/gp_foreign_data.out +++ b/src/test/regress/expected/gp_foreign_data.out @@ -26,18 +26,14 @@ CREATE FOREIGN TABLE ft3 ( CREATE FOREIGN TABLE ft4 ( c1 int ) SERVER s0 OPTIONS (delimiter ',', mpp_execute 'all segments'); --- Test num_segments option -CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '3'); -CREATE FOREIGN TABLE ft5 ( - c1 int -) SERVER s1 OPTIONS (delimiter ',', mpp_execute 'all segments', num_segments '5'); -\d+ ft5 - Foreign table "public.ft5" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+---------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | integer | | | | | plain | | -Server: s1 -FDW options: (delimiter ',', mpp_execute 'all segments', num_segments '5') +-- CREATE FOREIGN SERVER WITH num_segments +CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '5'); +-- CHECK FOREIGN SERVER's OPTIONS +SELECT srvoptions FROM pg_foreign_server WHERE srvname = 's1'; + srvoptions +------------------ + {num_segments=5} +(1 row) --start_ignore DROP FOREIGN DATA WRAPPER dummy CASCADE; diff --git a/src/test/regress/sql/gp_foreign_data.sql b/src/test/regress/sql/gp_foreign_data.sql index a26f96f9d50..4db395743ad 100644 --- a/src/test/regress/sql/gp_foreign_data.sql +++ b/src/test/regress/sql/gp_foreign_data.sql @@ -2,6 +2,12 @@ -- Test foreign-data wrapper and server management. Cloudberry MPP specific -- +-- start_ignore +DROP SERVER s0 CASCADE; +DROP SERVER s1 CASCADE; +DROP FOREIGN DATA WRAPPER dummy CASCADE; +-- end_ignore + CREATE FOREIGN DATA WRAPPER dummy; COMMENT ON FOREIGN DATA WRAPPER dummy IS 'useless'; @@ -21,13 +27,8 @@ CREATE FOREIGN TABLE ft4 ( c1 int ) SERVER s0 OPTIONS (delimiter ',', mpp_execute 'all segments'); --- Test num_segments option -CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '3'); -CREATE FOREIGN TABLE ft5 ( - c1 int -) SERVER s1 OPTIONS (delimiter ',', mpp_execute 'all segments', num_segments '5'); -\d+ ft5 +-- CREATE FOREIGN SERVER WITH num_segments +CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '5'); ---start_ignore -DROP FOREIGN DATA WRAPPER dummy CASCADE; ---end_ignore +-- CHECK FOREIGN SERVER's OPTIONS +SELECT srvoptions FROM pg_foreign_server WHERE srvname = 's1'; From e2c88b8c01fe1183fbace8a0b1fd170397d5745a Mon Sep 17 00:00:00 2001 From: Adam Lee Date: Wed, 23 Feb 2022 13:07:28 +0800 Subject: [PATCH 20/24] Remove coverity pipeline It's out of date for quite a while, I also don't know if anyone uses them for Greenplum development. --- concourse/pipelines/pipeline_coverity.yml | 61 ------------- concourse/scripts/scan_with_coverity.bash | 101 ---------------------- concourse/tasks/scan_with_coverity.yml | 17 ---- 3 files changed, 179 deletions(-) delete mode 100644 concourse/pipelines/pipeline_coverity.yml delete mode 100755 concourse/scripts/scan_with_coverity.bash delete mode 100644 concourse/tasks/scan_with_coverity.yml diff --git a/concourse/pipelines/pipeline_coverity.yml b/concourse/pipelines/pipeline_coverity.yml deleted file mode 100644 index c055129a906..00000000000 --- a/concourse/pipelines/pipeline_coverity.yml +++ /dev/null @@ -1,61 +0,0 @@ -resource_types: -- name: gcs - type: registry-image - source: - repository: frodenas/gcs-resource - -resources: -- name: gpdb_src - type: git - source: - branch: ((gpdb-git-branch)) - uri: ((gpdb-git-remote)) - -- name: centos-coverity - type: registry-image - source: - repository: pivotaldata/centos-coverity - username: ((docker_username)) - password: ((docker_password)) - -- name: coverity_daily - type: time - source: - location: America/Los_Angeles - days: [Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday] - start: 4:00 AM - stop: 5:00 AM - -- name: libquicklz-centos7 - type: gcs - source: - bucket: ((gcs-bucket)) - json_key: ((concourse-gcs-resources-service-account-key))) - regexp: gp-internal-artifacts/centos7/libquicklz-(1\.5\.0-.*)-1.el7.x86_64.rpm - -- name: libquicklz-devel-centos7 - type: gcs - source: - bucket: ((gcs-bucket)) - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: gp-internal-artifacts/centos7/libquicklz-devel-(1\.5\.0-.*)-1.el7.x86_64.rpm - -jobs: - -- name: coverity_scan - plan: - - get: coverity_daily - trigger: true - - aggregate: - - get: gpdb_src - - get: centos-coverity - - get: libquicklz-installer - resource: libquicklz-centos7 - - get: libquicklz-devel-installer - resource: libquicklz-devel-centos7 - - task: scan_with_coverity - file: gpdb_src/concourse/tasks/scan_with_coverity.yml - image: centos-coverity - params: - COVERITY_TOKEN: ((coverity_token)) - COVERITY_EMAIL: ((coverity_email)) diff --git a/concourse/scripts/scan_with_coverity.bash b/concourse/scripts/scan_with_coverity.bash deleted file mode 100755 index 5fab1bb37f2..00000000000 --- a/concourse/scripts/scan_with_coverity.bash +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -l -set -exo pipefail - -BASE_DIR=$(pwd) -export GPDB_ARTIFACTS_DIR -GPDB_ARTIFACTS_DIR=$BASE_DIR/$OUTPUT_ARTIFACT_DIR - -function prep_env_for_centos() { - BLD_ARCH=rhel7_x86_64 - echo "Detecting java7 path ..." - java7_packages=($(rpm -qa | grep -F java-1.7)) - java7_bin="$(rpm -ql "${java7_packages[@]}" | grep /jre/bin/java$)" - alternatives --set java "$java7_bin" - export JAVA_HOME="${java7_bin/jre\/bin\/java/}" - ln -sf /usr/bin/xsubpp /usr/share/perl5/ExtUtils/xsubpp - - export PATH=${JAVA_HOME}/bin:${PATH} -} - -function generate_build_number() { - pushd gpdb_src - #Only if its git repro, add commit SHA as build number - # BUILD_NUMBER file is used by getversion file in GPDB to append to version - if [ -d .git ] ; then - echo "commit:$(git rev-parse HEAD)" > BUILD_NUMBER - fi - popd -} - -function make_sync_tools() { - pushd gpdb_src/gpAux - make sync_tools - # We have compiled LLVM with native zlib on CentOS6 and not from - # the zlib downloaded from artifacts. Therefore, remove the zlib - # downloaded from artifacts in order to use the native zlib. - find ext -name 'libz.*' -exec rm -f {} \; - popd -} - -function build_gpdb_and_scan_with_coverity() { - local cov_int_dir="$1" - - pushd gpdb_src/gpAux - cov-build --dir "$cov_int_dir" make BLD_TARGETS="gpdb" GPROOT=/usr/local - popd -} - -function upload_to_coverity() { - ( - set +x - local cov_int_base="$1" - local sha="$2" - local cov_int_tar="$cov_int_base"/cov-int.tgz - - tar czfp "$cov_int_tar" -C "$cov_int_base" cov-int - - response=$(curl --verbose \ - --progress-bar \ - --form token="$COVERITY_TOKEN" \ - --form email="$COVERITY_EMAIL" \ - --form file=@"$cov_int_tar" \ - --form version="$sha" \ - --form description="Generated by Concourse on https://gpdb.data.pivotal.ci/" \ - https://scan.coverity.com/builds?project=greenplum-db%2Fgpdb) - - ERROR_STRINGS=( - "quota for this project has been reached" - ) - - for ERR in "${ERROR_STRINGS[@]}"; do - if echo "$response" | grep -q "$ERR"; then - echo "Coverty returned: \"$response\"" - echo "Response matches following know error: \"$ERR\"" - exit 1 - fi - done - ) -} - -function install_deps_for_centos() { - # quicklz is proprietary code that we cannot put in our public Docker images. - rpm -i libquicklz-installer/libquicklz-*.rpm - rpm -i libquicklz-devel-installer/libquicklz-*.rpm -} - -function _main() { - install_deps_for_centos - prep_env_for_centos - generate_build_number - make_sync_tools - - /opt/prepare-coverity.bash - - mkdir -p "$GPDB_ARTIFACTS_DIR"/cov-int - build_gpdb_and_scan_with_coverity "$GPDB_ARTIFACTS_DIR"/cov-int - - sha=$(cd gpdb_src && git rev-parse HEAD) - upload_to_coverity "$GPDB_ARTIFACTS_DIR" "$sha" -} - -_main "$@" diff --git a/concourse/tasks/scan_with_coverity.yml b/concourse/tasks/scan_with_coverity.yml deleted file mode 100644 index b550e825d37..00000000000 --- a/concourse/tasks/scan_with_coverity.yml +++ /dev/null @@ -1,17 +0,0 @@ -platform: linux -image_resource: - type: registry-image - source: - repository: pivotaldata/centos-coverity -inputs: - - name: gpdb_src - - name: libquicklz-installer - - name: libquicklz-devel-installer -outputs: - - name: gpdb_coverity_artifacts -run: - path: gpdb_src/concourse/scripts/scan_with_coverity.bash -params: - OUTPUT_ARTIFACT_DIR: gpdb_coverity_artifacts - COVERITY_TOKEN: - COVERITY_EMAIL: From c17eab5165be43be7cd79f8d9a97196a75859c95 Mon Sep 17 00:00:00 2001 From: Annpurna Shahani <30636132+Annu149@users.noreply.github.com> Date: Fri, 25 Feb 2022 09:25:57 +0530 Subject: [PATCH 21/24] Changes to avoid gpstop errors when standby is not reachable. (#13062) * Changes to avoid gpstop errors when standby is not reachable. Added check for standby reachability before stopping the standby to avoid gpstop failures when standby is unreachable. Added behave test case for gpstop when standby is not reachable. Addressed review comments (Updated warning message) --- gpMgmt/bin/gpstop | 5 ++ gpMgmt/test/behave/mgmt_utils/gpstop.feature | 10 ++++ .../test/behave/mgmt_utils/steps/gpstart.py | 50 +++++++++++-------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/gpMgmt/bin/gpstop b/gpMgmt/bin/gpstop index fb852d94f2e..6862e42ba90 100755 --- a/gpMgmt/bin/gpstop +++ b/gpMgmt/bin/gpstop @@ -34,6 +34,7 @@ try: from gppylib.gp_era import GpEraFile from gppylib.operations.utils import ParallelOperation, RemoteOperation from gppylib.operations.rebalanceSegments import ReconfigDetectionSQLQueryCommand + from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -514,6 +515,10 @@ class GpStop: if self.gparray.standbyCoordinator: standby = self.gparray.standbyCoordinator + if get_unreachable_segment_hosts([standby.hostname], 1): + logger.warning("Standby is unreachable, skipping shutdown on standby") + return True + logger.info("Stopping coordinator standby host %s mode=%s" % (standby.hostname, self.mode)) try: cmd = SegmentStop("stopping coordinator standby", diff --git a/gpMgmt/test/behave/mgmt_utils/gpstop.feature b/gpMgmt/test/behave/mgmt_utils/gpstop.feature index e5ea0c41b3e..7ff5b86ae7c 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpstop.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpstop.feature @@ -27,3 +27,13 @@ Feature: gpstop behave tests And gpstop should print "There were 1 user connections at the start of the shutdown" to stdout And gpstop should print "'\(s\)mart_mode', '\(f\)ast_mode', '\(i\)mmediate_mode'" to stdout Then gpstop should return a return code of 0 + + @demo_cluster + Scenario: gpstop succeeds even if the standby host is unreachable + Given the database is running + And the catalog has a standby coordinator entry + When the standby host is made unreachable + And the user runs "gpstop -a" + Then gpstop should print "Standby is unreachable, skipping shutdown on standby" to stdout + And gpstop should return a return code of 0 + And the standby host is made reachable diff --git a/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py b/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py index cfb23d95fa8..2046dbcae4f 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py @@ -37,30 +37,36 @@ def change_hostname(content, preferred_role, hostname): def impl(context): change_hostname(-1, 'm', 'invalid_host') - def cleanup(context): - """ - Reverses the above SQL by starting up in coordinator-only utility mode. Since - the standby host is incorrect, a regular gpstart call won't work. - """ - utils.stop_database_if_started(context) - - subprocess.check_call(['gpstart', '-am']) - _run_sql(""" - SET allow_system_table_mods='true'; - UPDATE gp_segment_configuration - SET hostname = coordinator.hostname, - address = coordinator.address - FROM ( - SELECT hostname, address - FROM gp_segment_configuration - WHERE content = -1 and role = 'p' - ) coordinator - WHERE content = -1 AND role = 'm' - """, {'gp_role': 'utility'}) - subprocess.check_call(['gpstop', '-am']) - context.add_cleanup(cleanup, context) +@when('the standby host is made reachable') +@then('the standby host is made reachable') +def impl(context): + cleanup(context) + +""" +Reverses the changes done by change_hostname() function by starting up cluster in master-only utility mode. +Since the standby host is incorrect, a regular gpstart call won't work. +""" +def cleanup(context): + + utils.stop_database_if_started(context) + + subprocess.check_call(['gpstart', '-am']) + _run_sql(""" + SET allow_system_table_mods='true'; + UPDATE gp_segment_configuration + SET hostname = coordinator.hostname, + address = coordinator.address + FROM ( + SELECT hostname, address + FROM gp_segment_configuration + WHERE content = -1 and role = 'p' + ) coordinator + WHERE content = -1 AND role = 'm' + """, {'gp_role': 'utility'}) + subprocess.check_call(['gpstop', '-am']) + def _handle_sigpipe(): """ Work around https://bugs.python.org/issue1615376, which is not fixed until From 7f24ced9ea04e68352d164067d5430e03bbae822 Mon Sep 17 00:00:00 2001 From: Xing Guo Date: Mon, 28 Feb 2022 15:08:30 +0800 Subject: [PATCH 22/24] Set statement_mem to a larger value in tests to make pipelines happy. (#13144) We observed greenplum built without cassert couldn't pass tests. Because when the gpdb is configured with --enable-cassert, the minimum statement_mem it accepts is 50kB while if it's configured without --enable-cassert, the minimum statement_mem it accepts is 1000kB (See: backend/utils/misc/guc_gp.c). This patch is trying to make pipelines happy. --- src/test/regress/expected/misc_jiras.out | 6 +++--- .../regress/expected/statement_mem_for_windowagg.out | 12 ++++++------ src/test/regress/sql/misc_jiras.sql | 4 ++-- src/test/regress/sql/statement_mem_for_windowagg.sql | 12 ++++++------ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/test/regress/expected/misc_jiras.out b/src/test/regress/expected/misc_jiras.out index ebe7446972e..ab418abb849 100644 --- a/src/test/regress/expected/misc_jiras.out +++ b/src/test/regress/expected/misc_jiras.out @@ -12,10 +12,10 @@ create schema misc_jiras; -- create table misc_jiras.t1 (c1 int, c2 text, c3 smallint) distributed by (c1); insert into misc_jiras.t1 select i % 13, md5(i::text), i % 3 - from generate_series(1, 20000) i; + from generate_series(1, 40000) i; -- tuplestore in windowagg uses statement_mem to control the in-memory data size, -- set a small value to trigger the spilling. -set statement_mem to '512kB'; +set statement_mem to '1024kB'; set extra_float_digits=0; -- the last decimal digits are somewhat random -- Inject fault at 'winagg_after_spool_tuples' to show that the tuplestore spills -- to disk. @@ -44,7 +44,7 @@ NOTICE: winagg: tuplestore spilled to disk (seg1 slice1 127.0.0.1:7003 pid=547 NOTICE: winagg: tuplestore spilled to disk (seg2 slice1 127.0.0.1:7004 pid=54721) sum --------- - 10006.5 + 20006.5 (1 row) SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) diff --git a/src/test/regress/expected/statement_mem_for_windowagg.out b/src/test/regress/expected/statement_mem_for_windowagg.out index a2e44474ec3..d41610f6a30 100644 --- a/src/test/regress/expected/statement_mem_for_windowagg.out +++ b/src/test/regress/expected/statement_mem_for_windowagg.out @@ -1,11 +1,11 @@ CREATE TABLE dummy_table(x int, y int) DISTRIBUTED BY (y); -INSERT INTO dummy_table SELECT generate_series(0, 10000), 0; -INSERT INTO dummy_table SELECT generate_series(0, 10000), 3; -INSERT INTO dummy_table SELECT generate_series(0, 10000), 10; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 0; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 3; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 10; -- 1. Test that if we set statement_mem to a larger value, the tuplestore -- for caching the tuples in partition used in WindowAgg is able to be fitted -- in memory. -SET statement_mem TO '2048kB'; +SET statement_mem TO '4096kB'; SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content>=0; gp_inject_fault @@ -89,7 +89,7 @@ SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) -- 3. Test that if we set statement_mem to a larger value, the tuplesort -- operation in DISTINCT-qualified WindowAgg is able to be fitted in memory. -SET statement_mem TO '1024kB'; +SET statement_mem TO '4096kB'; SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content>=0; gp_inject_fault_infinite @@ -131,7 +131,7 @@ SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) -- 4. Test that if we set statement_mem to a smaller value, the tuplesort -- operation in DISTINCT-qualified WindowAgg will be spilled to disk. -SET statement_mem TO '128kB'; +SET statement_mem TO '1024kB'; SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content>=0; gp_inject_fault_infinite diff --git a/src/test/regress/sql/misc_jiras.sql b/src/test/regress/sql/misc_jiras.sql index 1536496d505..4e96bc11e52 100644 --- a/src/test/regress/sql/misc_jiras.sql +++ b/src/test/regress/sql/misc_jiras.sql @@ -13,11 +13,11 @@ create schema misc_jiras; create table misc_jiras.t1 (c1 int, c2 text, c3 smallint) distributed by (c1); insert into misc_jiras.t1 select i % 13, md5(i::text), i % 3 - from generate_series(1, 20000) i; + from generate_series(1, 40000) i; -- tuplestore in windowagg uses statement_mem to control the in-memory data size, -- set a small value to trigger the spilling. -set statement_mem to '512kB'; +set statement_mem to '1024kB'; set extra_float_digits=0; -- the last decimal digits are somewhat random diff --git a/src/test/regress/sql/statement_mem_for_windowagg.sql b/src/test/regress/sql/statement_mem_for_windowagg.sql index 28339d55ba7..5c4b96a1761 100644 --- a/src/test/regress/sql/statement_mem_for_windowagg.sql +++ b/src/test/regress/sql/statement_mem_for_windowagg.sql @@ -1,12 +1,12 @@ CREATE TABLE dummy_table(x int, y int) DISTRIBUTED BY (y); -INSERT INTO dummy_table SELECT generate_series(0, 10000), 0; -INSERT INTO dummy_table SELECT generate_series(0, 10000), 3; -INSERT INTO dummy_table SELECT generate_series(0, 10000), 10; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 0; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 3; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 10; -- 1. Test that if we set statement_mem to a larger value, the tuplestore -- for caching the tuples in partition used in WindowAgg is able to be fitted -- in memory. -SET statement_mem TO '2048kB'; +SET statement_mem TO '4096kB'; SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content>=0; @@ -30,7 +30,7 @@ SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) -- 3. Test that if we set statement_mem to a larger value, the tuplesort -- operation in DISTINCT-qualified WindowAgg is able to be fitted in memory. -SET statement_mem TO '1024kB'; +SET statement_mem TO '4096kB'; SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content>=0; @@ -42,7 +42,7 @@ SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) -- 4. Test that if we set statement_mem to a smaller value, the tuplesort -- operation in DISTINCT-qualified WindowAgg will be spilled to disk. -SET statement_mem TO '128kB'; +SET statement_mem TO '1024kB'; SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content>=0; From 6e0d8ef6ac6487ab73d0faba39890b27cf234d52 Mon Sep 17 00:00:00 2001 From: Xing Guo Date: Mon, 28 Feb 2022 18:13:34 +0800 Subject: [PATCH 23/24] Trying to make parallel_retrieve_cursor/fault_inject stable. (#13141) This patch is trying to make isolation2/parallel_retrieve_cursor/fault_inject stable by replacing 'sleep' fault with 'suspend' fault. Below is the regression.diffs file fetched from the PR pipeline. ```diff diff -I HINT: -I CONTEXT: -I GP_IGNORE: -U3 /tmp/build/d62a0504/gpdb_src/src/test/isolation2/expected/parallel_retrieve_cursor/fault_inject.out /tmp/build/d62a0504/gpdb_src/src/test/isolation2/results/parallel_retrieve_cursor/fault_inject.out --- /tmp/build/d62a0504/gpdb_src/src/test/isolation2/expected/parallel_retrieve_cursor/fault_inject.out 2022-02-24 16:18:00.128491568 +0000 +++ /tmp/build/d62a0504/gpdb_src/src/test/isolation2/results/parallel_retrieve_cursor/fault_inject.out 2022-02-24 16:18:00.940556278 +0000 @@ -684,6 +684,3256 @@ READY (1 row) 2R&: @pre_run 'set_endpoint_variable @ENDPOINT6': RETRIEVE ALL FROM ENDPOINT "@ENDPOINT6"; +FAILED: Forked command is not blocking; got output: a +------- + 5 + 6 + 9 + ... + 9999 + 10000 +(3247 rows) 1U: SELECT state FROM gp_segment_endpoints() WHERE cursorname='c1'; state @@ -699,7 +3949,7 @@ 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:LINE_NUM) 2R<: <... completed> -ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:LINE_NUM) +FAILED: Execution failed 1<: <... completed> FAILED: Execution failed ``` --- .../fault_inject.source | 39 +++++++--- .../fault_inject.source | 71 ++++++++++++++++--- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source b/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source index 3e592b5968c..c04643d14f8 100644 --- a/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source +++ b/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source @@ -114,8 +114,8 @@ insert into t1 select generate_series(1,100); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 2); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 3); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 4); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 5, 5, 0, 3::smallint); 1: BEGIN; @@ -132,8 +132,14 @@ insert into t1 select generate_series(1,100); 1U: SELECT state FROM gp_segment_endpoints() WHERE cursorname='c1'; 1R: @pre_run 'set_endpoint_variable @ENDPOINT5': RETRIEVE ALL FROM ENDPOINT "@ENDPOINT5"; +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 2); +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 4); + 1<: +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + 0R<: 2R<: @@ -144,9 +150,9 @@ insert into t1 select generate_series(1,100); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 4); -- Test6: close PARALLEL RETRIEVE CURSOR during retrieve -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 3::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 3::smallint); 1: BEGIN; 1: DECLARE c1 PARALLEL RETRIEVE CURSOR FOR SELECT * from t1; @@ -164,6 +170,10 @@ insert into t1 select generate_series(1,100); 1: SELECT * FROM gp_wait_parallel_retrieve_cursor('c1', 0); 1: CLOSE c1; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 3); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + 0R<: 1R<: 2R<: @@ -181,22 +191,22 @@ DROP TABLE IF EXISTS t2; CREATE TABLE t2 (a INT) DISTRIBUTED by (a); insert into t2 select generate_series(1,10000); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 900, 900, 2, dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 900, 900, 0, dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 800, 800, 2, dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 800, 800, 0, dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; @@ -216,6 +226,13 @@ insert into t2 select generate_series(1,10000); 1<: +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) + FROM gp_segment_configuration + WHERE content=0 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) + FROM gp_segment_configuration + WHERE content=2 AND role='p'; + 0R<: 2R<: diff --git a/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source b/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source index a047a0c8294..044e288796b 100644 --- a/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source +++ b/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source @@ -457,12 +457,12 @@ ROLLBACK ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); gp_inject_fault ----------------- Success: @@ -506,9 +506,31 @@ DECLARE 1R: @pre_run 'set_endpoint_variable @ENDPOINT5': RETRIEVE ALL FROM ENDPOINT "@ENDPOINT5"; ERROR: canceling statement due to user request +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 2); + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 4); + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) + 1<: <... completed> ERROR: canceling MPP operation: "Endpoint retrieve statement aborted" +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + gp_inject_fault +----------------- + Success: +(1 row) + 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:245) 2R<: <... completed> @@ -535,17 +557,17 @@ ROLLBACK (1 row) -- Test6: close PARALLEL RETRIEVE CURSOR during retrieve -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 3::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 3::smallint); gp_inject_fault ----------------- Success: @@ -590,6 +612,22 @@ DECLARE 1: CLOSE c1; CLOSE +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 3); + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + gp_inject_fault +----------------- + Success: +(1 row) + 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:245) 1R<: <... completed> @@ -628,32 +666,32 @@ CREATE insert into t2 select generate_series(1,10000); INSERT 10000 -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 900, 900, 2, dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 900, 900, 0, dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 800, 800, 2, dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 800, 800, 0, dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; gp_inject_fault ----------------- Success: @@ -695,6 +733,17 @@ ERROR: canceling statement due to user request 1<: <... completed> ERROR: canceling MPP operation: "Endpoint retrieve statement aborted" +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; + gp_inject_fault +----------------- + Success: +(1 row) + 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:245) 2R<: <... completed> From 7d58e77a497d89df005dc46960dae89f0c1c1910 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Mon, 13 May 2024 10:33:32 +0800 Subject: [PATCH 24/24] Fix Merge GPDB. Some codes do not work in CBDB after Merge from GPDB. Fix errors and etc. Authored-by: Zhang Mingli avamingli@gmail.com --- src/backend/optimizer/util/pathnode.c | 8 ++++---- .../regress/expected/aggregates_optimizer.out | 2 ++ src/test/regress/expected/gp_dqa_optimizer.out | 16 ++++++++++++++++ src/test/regress/sql/gp_foreign_data.sql | 6 ++++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 540d27abc48..9cf83acc338 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -3648,9 +3648,9 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, case FTEXECLOCATION_ALL_SEGMENTS: server = GetForeignServer(rel->serverid); if (server) - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments); + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments, 0); else - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount()); + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount(), 0); break; case FTEXECLOCATION_COORDINATOR: CdbPathLocus_MakeEntry(&(pathnode->path.locus)); @@ -3718,9 +3718,9 @@ create_foreign_join_path(PlannerInfo *root, RelOptInfo *rel, case FTEXECLOCATION_ALL_SEGMENTS: server = GetForeignServer(rel->serverid); if (server) - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments); + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments, 0); else - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount()); + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount(), 0); break; case FTEXECLOCATION_COORDINATOR: CdbPathLocus_MakeEntry(&(pathnode->path.locus)); diff --git a/src/test/regress/expected/aggregates_optimizer.out b/src/test/regress/expected/aggregates_optimizer.out index 634744680d9..87d3dd7ee53 100644 --- a/src/test/regress/expected/aggregates_optimizer.out +++ b/src/test/regress/expected/aggregates_optimizer.out @@ -2944,6 +2944,8 @@ INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: GPDB Expression type: Query Parameter not supported in DXL INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: GPDB Expression type: Query Parameter not supported in DXL +NOTICE: avg_transfn called with 1 +NOTICE: avg_transfn called with 3 NOTICE: avg_transfn called with 3 INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: GPDB Expression type: Query Parameter not supported in DXL diff --git a/src/test/regress/expected/gp_dqa_optimizer.out b/src/test/regress/expected/gp_dqa_optimizer.out index bcc734b3978..7c28cc1d7c2 100644 --- a/src/test/regress/expected/gp_dqa_optimizer.out +++ b/src/test/regress/expected/gp_dqa_optimizer.out @@ -2338,12 +2338,16 @@ insert into dqa_f3 values ('123', 2), ('213', 0), ('231', 2), ('312', 0), ('321' -- -> Seq Scan on public.dqa_f3 -- Output: b, a, (b)::text select count(distinct (b)::text) as b, count(distinct (a)::text) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::text) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------ Finalize Aggregate @@ -2369,12 +2373,16 @@ explain (verbose, costs off) select count(distinct (b)::text) as b, count(distin -- Case 2: Same as the above one, but convert the type of column 'a' to 'varchar' via binary-compatible types. select count(distinct (b)::text) as b, count(distinct (a)::text::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::text::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -2410,12 +2418,16 @@ explain (verbose, costs off) select count(distinct (b)::text) as b, count(distin -- -> Seq Scan on public.dqa_f3 -- Output: b, a, (b)::text, (a)::integer select count(distinct (b)::text) as b, count(distinct (a)::int) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::int) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN --------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -2442,12 +2454,16 @@ explain (verbose, costs off) select count(distinct (b)::text) as b, count(distin -- Case 4: When converting the type of column 'a' from 'varchar' to 'int' to 'varchar', TupleSplit should generate an additional -- column '(a)::integer::varchar' as part of hash-key in Redistribute-Motion. select count(distinct (b)::text) as b, count(distinct (a)::int::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::int::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------------------------------ Finalize Aggregate diff --git a/src/test/regress/sql/gp_foreign_data.sql b/src/test/regress/sql/gp_foreign_data.sql index 4db395743ad..23c630797da 100644 --- a/src/test/regress/sql/gp_foreign_data.sql +++ b/src/test/regress/sql/gp_foreign_data.sql @@ -32,3 +32,9 @@ CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '5'); -- CHECK FOREIGN SERVER's OPTIONS SELECT srvoptions FROM pg_foreign_server WHERE srvname = 's1'; + +-- start_ignore +DROP SERVER s0 CASCADE; +DROP SERVER s1 CASCADE; +DROP FOREIGN DATA WRAPPER dummy CASCADE; +-- end_ignore