diff --git a/gpMgmt/bin/gpcheckcat b/gpMgmt/bin/gpcheckcat index ea8ed134180..4150acbae4e 100755 --- a/gpMgmt/bin/gpcheckcat +++ b/gpMgmt/bin/gpcheckcat @@ -1283,6 +1283,10 @@ def checkTableMissingEntry(cat): if catname == "gp_segment_configuration": return + # Skip gp_matview_aux or gp_matview_tables + if catname == "gp_matview_aux" or catname == "gp_matview_tables": + return + # skip shared/non-shared tables if GV.opt['-S']: if re.match("none", GV.opt['-S'], re.I) and isShared: @@ -1533,6 +1537,10 @@ def checkTableInconsistentEntry(cat): if catname == "gp_segment_configuration" or catname == "pg_appendonly": return + # Skip gp_matview_aux or gp_matview_tables + if catname == "gp_matview_aux" or catname == "gp_matview_tables": + return + # skip shared/non-shared tables if GV.opt['-S']: if re.match("none", GV.opt['-S'], re.I) and isShared: diff --git a/gpMgmt/bin/gppylib/gpcatalog.py b/gpMgmt/bin/gppylib/gpcatalog.py index a518878559d..1a7325daff6 100644 --- a/gpMgmt/bin/gppylib/gpcatalog.py +++ b/gpMgmt/bin/gppylib/gpcatalog.py @@ -33,7 +33,9 @@ class GPCatalogException(Exception): 'pg_statistic_ext', 'pg_statistic_ext_data', 'gp_partition_template', # GPDB_12_MERGE_FIXME: is gp_partition_template intentionally missing from segments? - 'pg_event_trigger' + 'pg_event_trigger', + 'gp_matview_aux', + 'gp_matview_tables', ] # Hard coded tables that have different values on every segment diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index c3ccbdc18e4..4670b33372c 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -53,6 +53,7 @@ OBJS += pg_extprotocol.o \ oid_dispatch.o aocatalog.o storage_tablespace.o storage_database.o \ storage_tablespace_twophase.o storage_tablespace_xact.o \ gp_partition_template.o pg_task.o pg_task_run_history.o \ + gp_matview_aux.o \ pg_directory_table.o storage_directory_table.o CATALOG_JSON:= $(addprefix $(top_srcdir)/gpMgmt/bin/gppylib/data/, $(addsuffix .json,$(GP_MAJORVERSION))) @@ -94,6 +95,8 @@ CATALOG_HEADERS := \ pg_sequence.h pg_publication.h pg_publication_rel.h pg_subscription.h \ pg_subscription_rel.h gp_partition_template.h pg_task.h pg_task_run_history.h \ pg_profile.h pg_password_history.h pg_directory_table.h gp_storage_server.h \ + gp_matview_aux.h \ + gp_matview_tables.h \ gp_storage_user_mapping.h USE_INTERNAL_FTS_FOUND := $(if $(findstring USE_INTERNAL_FTS,$(CFLAGS)),true,false) diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c index 8c4c7023921..53a246fc3e2 100644 --- a/src/backend/catalog/catalog.c +++ b/src/backend/catalog/catalog.c @@ -76,6 +76,8 @@ #include "catalog/pg_stat_last_shoperation.h" #include "catalog/pg_statistic.h" #include "catalog/pg_trigger.h" +#include "catalog/gp_matview_aux.h" +#include "catalog/gp_matview_tables.h" #include "cdb/cdbvars.h" #include "catalog/gp_indexing.h" @@ -551,6 +553,16 @@ IsSharedRelation(Oid relationId) return true; } + /* materialized view aux and its indexes */ + if (relationId == GpMatviewAuxId || + relationId == GpMatviewAuxMvoidIndexId || + relationId == GpMatviewAuxMvnameIndexId || + relationId == GpMatviewAuxDatastatusIndexId || + relationId == GpMatviewTablesId || + relationId == GpMatviewTablesMvRelIndexId || + relationId == GpMatviewTablesRelIndexId) + return true; + /* warehouse table and its indexes */ if (relationId == GpWarehouseRelationId || relationId == GpWarehouseOidIndexId || diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 6be702e488b..ceb5d51ad62 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -100,6 +100,7 @@ #include "catalog/pg_profile.h" #include "catalog/pg_password_history.h" #include "commands/tablecmds.h" +#include "catalog/gp_matview_aux.h" /* @@ -209,7 +210,8 @@ static const Oid object_classes[] = { StorageServerRelationId, /* OCLASS_STORAGE_SERVER */ StorageUserMappingRelationId, /* OCLASS_STORAGE_USER_MAPPING */ ExtprotocolRelationId, /* OCLASS_EXTPROTOCOL */ - TaskRelationId /* OCLASS_TASK */ + GpMatviewAuxId, /* OCLASS_MATVIEW_AUX */ + TaskRelationId, /* OCLASS_TASK */ }; @@ -1554,6 +1556,10 @@ doDeletion(const ObjectAddress *object, int flags) RemoveTaskById(object->objectId); break; + case OCLASS_MATVIEW_AUX: + RemoveMatviewAuxEntry(object->objectId); + break; + case OCLASS_CAST: case OCLASS_COLLATION: case OCLASS_CONVERSION: @@ -2978,6 +2984,9 @@ getObjectClass(const ObjectAddress *object) case TaskRelationId: return OCLASS_TASK; + case GpMatviewAuxId: + return OCLASS_MATVIEW_AUX; + case DirectoryTableRelationId: return OCLASS_DIRTABLE; diff --git a/src/backend/catalog/gp_matview_aux.c b/src/backend/catalog/gp_matview_aux.c new file mode 100644 index 00000000000..66715417c13 --- /dev/null +++ b/src/backend/catalog/gp_matview_aux.c @@ -0,0 +1,444 @@ +/*------------------------------------------------------------------------- + * + * gp_matview_aux.c + * + * Portions Copyright (c) 2024-Present HashData, Inc. or its affiliates. + * + * + * IDENTIFICATION + * src/backend/catalog/gp_matview_aux.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "nodes/pg_list.h" +#include "nodes/parsenodes.h" +#include "access/htup.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "access/genam.h" +#include "catalog/dependency.h" +#include "catalog/gp_matview_aux.h" +#include "catalog/gp_matview_tables.h" +#include "catalog/pg_type.h" +#include "catalog/indexing.h" +#include "cdb/cdbvars.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/rel.h" +#include "utils/relcache.h" +#include "utils/syscache.h" +#include "utils/lsyscache.h" +#include "storage/lockdefs.h" +#include "optimizer/optimizer.h" +#include "parser/parsetree.h" + +static void InsertMatviewTablesEntries(Oid mvoid, List *relids); + +static void RemoveMatviewTablesEntries(Oid mvoid); + +static void SetMatviewAuxStatus_guts(Oid mvoid, char status); + +/* + * GetViewBaseRelids + * Get all base tables's oid of a query tree. + * Currently there is only one base table, but there should be + * distinct func on it later. Self join tables: t1 join t1, will + * get only one oid. + * + * Return NIL if the query we think it's useless. + */ +List* +GetViewBaseRelids(const Query *viewQuery) +{ + List *relids = NIL; + Node *mvjtnode; + + if ((viewQuery->commandType != CMD_SELECT) || + (viewQuery->rowMarks != NIL) || + (viewQuery->distinctClause != NIL) || + (viewQuery->scatterClause != NIL) || + (viewQuery->cteList != NIL) || + (viewQuery->groupingSets != NIL) || + (viewQuery->havingQual != NULL) || + (viewQuery->setOperations != NULL) || + viewQuery->hasWindowFuncs || + viewQuery->hasDistinctOn || + viewQuery->hasModifyingCTE || + viewQuery->groupDistinct || + (viewQuery->parentStmtType == PARENTSTMTTYPE_REFRESH_MATVIEW) || + viewQuery->hasSubLinks) + { + return NIL; + } + + /* As we will use views, make it strict to unmutable. */ + if (contain_mutable_functions((Node*)viewQuery)) + return NIL; + + if (list_length(viewQuery->jointree->fromlist) != 1) + return NIL; + + mvjtnode = (Node *) linitial(viewQuery->jointree->fromlist); + if (!IsA(mvjtnode, RangeTblRef)) + return NIL; + + RangeTblEntry *rte = rt_fetch(1, viewQuery->rtable); + if (rte->rtekind != RTE_RELATION) + return NIL; + + /* Only support normal relation now. */ + if (get_rel_relkind(rte->relid) != RELKIND_RELATION) + return NIL; + + relids = list_make1_oid(rte->relid); + + return relids; +} + +static void +add_view_dependency(Oid mvoid) +{ + ObjectAddress myself, referenced; + + ObjectAddressSet(myself, GpMatviewAuxId, mvoid); + ObjectAddressSet(referenced, RelationRelationId, mvoid); + recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO); +} + + +/* + * InsertMatviewAuxEntry + * We also insert gp_matview_tables entry here to maintain view. + */ +void +InsertMatviewAuxEntry(Oid mvoid, const Query *viewQuery, bool skipdata) +{ + Relation mvauxRel; + HeapTuple tup; + bool nulls[Natts_gp_matview_aux]; + Datum values[Natts_gp_matview_aux]; + List *relids; + NameData mvname; + + Assert(OidIsValid(mvoid)); + + /* Empty relids means the view is not supported now. */ + relids = GetViewBaseRelids(viewQuery); + if (relids == NIL) + return; + + mvauxRel = table_open(GpMatviewAuxId, RowExclusiveLock); + + MemSet(nulls, false, sizeof(nulls)); + MemSet(values, 0, sizeof(values)); + + values[Anum_gp_matview_aux_mvoid - 1] = ObjectIdGetDatum(mvoid); + + namestrcpy(&mvname, get_rel_name(mvoid)); + values[Anum_gp_matview_aux_mvname - 1] = NameGetDatum(&mvname); + + if (skipdata) + values[Anum_gp_matview_aux_datastatus - 1] = CharGetDatum(MV_DATA_STATUS_EXPIRED); + else + values[Anum_gp_matview_aux_datastatus - 1] = CharGetDatum(MV_DATA_STATUS_UP_TO_DATE); + + tup = heap_form_tuple(RelationGetDescr(mvauxRel), values, nulls); + + CatalogTupleInsert(mvauxRel, tup); + + add_view_dependency(mvoid); + + /* Install view tables entries.*/ + InsertMatviewTablesEntries(mvoid, relids); + + table_close(mvauxRel, RowExclusiveLock); + + return; +} + +/* + * Insert all relid oids for a materialized view. + */ +static void +InsertMatviewTablesEntries(Oid mvoid, List *relids) +{ + Relation mtRel; + HeapTuple tup; + bool nulls[Natts_gp_matview_tables]; + Datum values[Natts_gp_matview_tables]; + ListCell *lc; + + mtRel = table_open(GpMatviewTablesId, RowExclusiveLock); + MemSet(nulls, false, sizeof(nulls)); + MemSet(values, 0, sizeof(values)); + + foreach(lc, relids) + { + Oid relid = lfirst_oid(lc); + values[Anum_gp_matview_tables_relid - 1] = ObjectIdGetDatum(relid); + values[Anum_gp_matview_tables_mvoid - 1] = ObjectIdGetDatum(mvoid); + tup = heap_form_tuple(RelationGetDescr(mtRel), values, nulls); + CatalogTupleInsert(mtRel, tup); + } + + table_close(mtRel, RowExclusiveLock); +} + +/* + * RemoveMatviewAuxEntryOid + * Not all materialized views have a corresponding aux entry. + * We will skip those are certainly no used for AQUMV or + * their date is awlays not up to date(ex: has volatile functions). + */ +void +RemoveMatviewAuxEntry(Oid mvoid) +{ + Relation mvauxRel; + HeapTuple tup; + + mvauxRel = table_open(GpMatviewAuxId, RowExclusiveLock); + + tup = SearchSysCache1(MVAUXOID, ObjectIdGetDatum(mvoid)); + + /* This clould happen, not all materialized views have an aux entry. */ + if (!HeapTupleIsValid(tup)) + { + ReleaseSysCache(tup); + table_close(mvauxRel, RowExclusiveLock); + return; + } + + CatalogTupleDelete(mvauxRel, &tup->t_self); + + ReleaseSysCache(tup); + + RemoveMatviewTablesEntries(mvoid); + + table_close(mvauxRel, RowExclusiveLock); + + return; +} + +static void +RemoveMatviewTablesEntries(Oid mvoid) +{ + Relation mtRel; + CatCList *catlist; + int i; + + mtRel = table_open(GpMatviewTablesId, RowExclusiveLock); + + catlist = SearchSysCacheList1(MVTABLESMVRELOID, ObjectIdGetDatum(mvoid)); + for (i = 0; i < catlist->n_members; i++) + { + HeapTuple tuple = &catlist->members[i]->tuple; + + /* This shouldn't happen, in case for that. */ + if (!HeapTupleIsValid(tuple)) + continue; + + CatalogTupleDelete(mtRel, &tuple->t_self); + } + + ReleaseSysCacheList(catlist); + + table_close(mtRel, RowExclusiveLock); + + return; +} + +/* + * Set all relative materialized views status if + * the underlying table's data is changed. + */ +void +SetRelativeMatviewAuxStatus(Oid relid, char status) +{ + Relation mvauxRel; + Relation mtRel; + HeapTuple tup; + ScanKeyData key; + SysScanDesc desc; + + mvauxRel = table_open(GpMatviewAuxId, RowExclusiveLock); + + /* Find all mvoids have relid */ + mtRel = table_open(GpMatviewTablesId, AccessShareLock); + ScanKeyInit(&key, + Anum_gp_matview_tables_relid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + desc = systable_beginscan(mtRel, + GpMatviewTablesRelIndexId, + true, + NULL, 1, &key); + while (HeapTupleIsValid(tup = systable_getnext(desc))) + { + Form_gp_matview_tables mt = (Form_gp_matview_tables) GETSTRUCT(tup); + /* Update mv aux status. */ + SetMatviewAuxStatus_guts(mt->mvoid, status); + } + + systable_endscan(desc); + table_close(mtRel, AccessShareLock); + table_close(mvauxRel, RowExclusiveLock); +} + +void +SetMatviewAuxStatus(Oid mvoid, char status) +{ + Relation mvauxRel; + mvauxRel = table_open(GpMatviewAuxId, RowExclusiveLock); + SetMatviewAuxStatus_guts(mvoid, status); + table_close(mvauxRel, RowExclusiveLock); +} + +/* + * This requires caller has opened the table and + * holds proper locks. + */ +static void +SetMatviewAuxStatus_guts(Oid mvoid, char status) +{ + HeapTuple tuple; + HeapTuple newtuple; + Relation mvauxRel; + Datum valuesAtt[Natts_gp_matview_aux]; + bool nullsAtt[Natts_gp_matview_aux]; + bool replacesAtt[Natts_gp_matview_aux]; + + switch (status) + { + case MV_DATA_STATUS_UP_TO_DATE: + case MV_DATA_STATUS_UP_REORGANIZED: + case MV_DATA_STATUS_EXPIRED: + case MV_DATA_STATUS_EXPIRED_INSERT_ONLY: + break; + default: + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid status: %c", status))); + } + + if (!IS_QUERY_DISPATCHER()) + { + ereport(WARNING, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("try to modify matview catalog outside QD"))); + return; + } + + tuple = SearchSysCacheCopy1(MVAUXOID, ObjectIdGetDatum(mvoid)); + + /* This could happen, Refresh a matview we didn't install in aux table. */ + if (!HeapTupleIsValid(tuple)) + return; + + Form_gp_matview_aux auxform = (Form_gp_matview_aux) GETSTRUCT(tuple); + + /* Nothing to do. */ + if (auxform->datastatus == status) + return; + + /* handle incomming insert only stauts */ + if (status == MV_DATA_STATUS_EXPIRED_INSERT_ONLY) + { + switch (auxform->datastatus) + { + case MV_DATA_STATUS_EXPIRED: + return; /* keep expired. */ + case MV_DATA_STATUS_UP_TO_DATE: + break; /* just update. */ + case MV_DATA_STATUS_UP_REORGANIZED: + { + /* + * Insert comes after VACUUM FULL or CLUSTER. + * There are pages reorganized and more date inserted. + * Neighter we can use this view as logically up to date + * nor fetch Delta pages on base table. + */ + status = MV_DATA_STATUS_EXPIRED; + break; + } + default: + Assert(false); /* should not get here. */ + break; + } + } + + /* handle incomming reorganized stauts */ + if (status == MV_DATA_STATUS_UP_REORGANIZED) + { + switch (auxform->datastatus) + { + case MV_DATA_STATUS_EXPIRED: + return; /* keep expired. */ + case MV_DATA_STATUS_UP_TO_DATE: + break; /* just update. */ + case MV_DATA_STATUS_EXPIRED_INSERT_ONLY: + { + /* + * Reorganize pages, Delta pages can not be + * fetched anymore. + */ + status = MV_DATA_STATUS_EXPIRED; + break; + } + default: + Assert(false); /* should not get here. */ + break; + } + } + + MemSet(valuesAtt, 0, sizeof(valuesAtt)); + MemSet(nullsAtt, false, sizeof(nullsAtt)); + MemSet(replacesAtt, false, sizeof(replacesAtt)); + + replacesAtt[Anum_gp_matview_aux_datastatus -1] = true; + valuesAtt[Anum_gp_matview_aux_datastatus - 1] = CharGetDatum(status); + + mvauxRel = table_open(GpMatviewAuxId, NoLock); + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(mvauxRel), + valuesAtt, nullsAtt, replacesAtt); + + CatalogTupleUpdate(mvauxRel, &newtuple->t_self, newtuple); + heap_freetuple(newtuple); + table_close(mvauxRel, NoLock); +} + +/* + * For SERVERLESS: + * Could view be used for Append Agg plan? + * This is only used for Incremental Append Agg plan purpose. + * + * If staus is insert only, we could expand Append Agg plan + * with the view. + * If status is up to date, we still do that where SeqScan on + * base table should get zero tuples. + * But VACUUM FULL and CLUTER commands will change the pages of + * base table, we could not fetch a Delta pages for Delta SeqScan. + */ +bool +MatviewUsableForAppendAgg(Oid mvoid) +{ + HeapTuple mvauxtup = SearchSysCacheCopy1(MVAUXOID, ObjectIdGetDatum(mvoid)); + Form_gp_matview_aux auxform = (Form_gp_matview_aux) GETSTRUCT(mvauxtup); + return ((auxform->datastatus == MV_DATA_STATUS_UP_TO_DATE) || + (auxform->datastatus == MV_DATA_STATUS_EXPIRED_INSERT_ONLY)); +} + +/* + * Is the view data up to date? + * In most cases, we should use this function to check if view + * data status is up to date. + * + * We include the cases VACUUM FULL/CLUSTER on base tables. + * Their data is logically not changed. + */ +bool +MatviewIsGeneralyUpToDate(Oid mvoid) +{ + HeapTuple mvauxtup = SearchSysCacheCopy1(MVAUXOID, ObjectIdGetDatum(mvoid)); + Form_gp_matview_aux auxform = (Form_gp_matview_aux) GETSTRUCT(mvauxtup); + return ((auxform->datastatus == MV_DATA_STATUS_UP_TO_DATE) || + (auxform->datastatus == MV_DATA_STATUS_UP_REORGANIZED)); +} diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 3183d113696..26eaf197f34 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -113,6 +113,7 @@ #include "utils/timestamp.h" #include "catalog/gp_indexing.h" +#include "catalog/gp_matview_aux.h" static void MetaTrackAddUpdInternal(Oid classid, Oid objoid, @@ -3953,6 +3954,10 @@ heap_truncate_one_rel(Relation rel) /* If the relation has indexes, truncate the indexes too */ RelationTruncateIndexes(rel); + /* update view info */ + if (IS_QD_OR_SINGLENODE()) + SetRelativeMatviewAuxStatus(RelationGetRelid(rel), MV_DATA_STATUS_EXPIRED); + /* If there is a toast table, truncate that too */ toastrelid = rel->rd_rel->reltoastrelid; if (OidIsValid(toastrelid)) diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 852af0f6759..0989cecc72a 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -4150,6 +4150,9 @@ getObjectDescription(const ObjectAddress *object, bool missing_ok) break; } + case OCLASS_MATVIEW_AUX: + break; + case OCLASS_STORAGE_SERVER: { StorageServer *srv; @@ -4781,6 +4784,10 @@ getObjectTypeDescription(const ObjectAddress *object, bool missing_ok) appendStringInfoString(&buffer, "directory table"); break; + case OCLASS_MATVIEW_AUX: + appendStringInfoString(&buffer, "matview_aux"); + break; + case OCLASS_STORAGE_SERVER: appendStringInfoString(&buffer, "storage server"); break; @@ -6197,6 +6204,9 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_MATVIEW_AUX: + break; + default: { struct CustomObjectClass *coc; diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index df43eb8e69b..ea774dbe8a1 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -734,6 +734,7 @@ AlterObjectNamespace_oid(Oid classId, Oid objid, Oid nspOid, case OCLASS_TASK: case OCLASS_PROFILE: case OCLASS_PASSWORDHISTORY: + case OCLASS_MATVIEW_AUX: case OCLASS_STORAGE_SERVER: case OCLASS_STORAGE_USER_MAPPING: /* ignore object types that don't have schema-qualified names */ diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index e0f76612b34..320d4ae57d9 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -30,6 +30,7 @@ #include "access/xlog.h" #include "catalog/catalog.h" #include "catalog/dependency.h" +#include "catalog/gp_matview_aux.h" #include "catalog/heap.h" #include "catalog/index.h" #include "catalog/namespace.h" @@ -219,6 +220,21 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) GetAssignedOidsForDispatch(), NULL); } + + if (IS_QD_OR_SINGLENODE()) + { + /* + * Update view status. + * In principle, CLUSTER command won't change the ligical data of + * a table, it may change the physical pages by index. + * But for Append Agg Plan in SERVERLESS mode, we need to fetch + * delta tuples from base table which requires the ability of storage + * to distint the pages instead, since latest relative materialized + * view REFRESH. + */ + SetRelativeMatviewAuxStatus(tableOid, MV_DATA_STATUS_UP_REORGANIZED); + + } } else { @@ -284,6 +300,9 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) GetAssignedOidsForDispatch(), NULL); } + /* See comments above. */ + if (IS_QD_OR_SINGLENODE()) + SetRelativeMatviewAuxStatus(rvtc->tableOid, MV_DATA_STATUS_UP_REORGANIZED); PopActiveSnapshot(); CommitTransactionCommand(); diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index a4988ef87fd..def55df85d6 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -56,6 +56,7 @@ #include "access/external.h" #include "access/url.h" #include "catalog/catalog.h" +#include "catalog/gp_matview_aux.h" #include "catalog/namespace.h" #include "catalog/pg_extprotocol.h" #include "cdb/cdbappendonlyam.h" @@ -442,9 +443,17 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, *processed = CopyFrom(cstate); /* copy from file to database */ /* Handle copy to replicated table returns processed number */ - if (Gp_role == GP_ROLE_DISPATCH && cstate->rel->rd_cdbpolicy && - cstate->rel->rd_cdbpolicy->ptype == POLICYTYPE_REPLICATED) + if (Gp_role == GP_ROLE_DISPATCH && + GpPolicyIsReplicated(cstate->rel->rd_cdbpolicy)) *processed = *processed / cstate->rel->rd_cdbpolicy->numsegments; + + /* + * Update view info if we actualy copy data from other place. + */ + if (IS_QD_OR_SINGLENODE() && *processed > 0) + { + SetRelativeMatviewAuxStatus(relid, MV_DATA_STATUS_EXPIRED_INSERT_ONLY); + } } PG_CATCH(); { diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index a91e85e8dbf..1f90b2741bb 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -70,6 +70,7 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" +#include "catalog/gp_matview_aux.h" #include "catalog/oid_dispatch.h" #include "cdb/cdbappendonlyam.h" #include "cdb/cdbaocsam.h" @@ -513,11 +514,21 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, /* Restore userid and security context */ SetUserIdAndSecContext(save_userid, save_sec_context); + Oid matviewOid = address.objectId; + Relation matviewRel = table_open(matviewOid, NoLock); + + /* + * Record materialized view aux entry. + * This is used to check if a materialized view's meta data, + * ex: data is up to date and etc. + * The info is used for expanding Incremental Appedn Agg Plan + * and Answer Query Using Materialized Views. + */ + if (IS_QD_OR_SINGLENODE()) + InsertMatviewAuxEntry(matviewOid, (Query* )into->viewQuery, into->skipData); + if (into->ivm) { - Oid matviewOid = address.objectId; - Relation matviewRel = table_open(matviewOid, NoLock); - /* * Mark relisivm field, if it's a matview and into->ivm is true. */ @@ -529,8 +540,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, /* Create triggers on incremental maintainable materialized view */ CreateIvmTriggersOnBaseTables(query_immv, matviewOid); } - table_close(matviewRel, NoLock); } + table_close(matviewRel, NoLock); } { diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 0b984fbb431..e821e00bab0 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -1028,6 +1028,7 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_ROLE: case OCLASS_PROFILE: case OCLASS_PASSWORDHISTORY: + case OCLASS_MATVIEW_AUX: case OCLASS_STORAGE_SERVER: case OCLASS_STORAGE_USER_MAPPING: /* no support for global objects */ diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index a662bc1b0e4..c0a4e72d4bc 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -24,6 +24,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/gp_matview_aux.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/oid_dispatch.h" @@ -470,6 +471,18 @@ ExecRefreshMatView(RefreshMatViewStmt *stmt, const char *queryString, */ SetMatViewPopulatedState(matviewRel, !stmt->skipData); + if (IS_QD_OR_SINGLENODE()) + { + /* + * Update view info: + * It's actually a TRUNCATE command if skipData is true. + */ + if (stmt->skipData) + SetMatviewAuxStatus(RelationGetRelid(matviewRel), MV_DATA_STATUS_EXPIRED); + else + SetMatviewAuxStatus(RelationGetRelid(matviewRel), MV_DATA_STATUS_UP_TO_DATE); + } + /* * The stored query was rewritten at the time of the MV definition, but * has not been scribbled on by the planner. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index e43db15714e..19414729f1d 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -129,6 +129,7 @@ #include "access/bitmap_private.h" #include "access/external.h" #include "catalog/aocatalog.h" +#include "catalog/gp_matview_aux.h" #include "catalog/oid_dispatch.h" #include "nodes/altertablenodes.h" #include "cdb/cdbdisp.h" @@ -2449,6 +2450,10 @@ ExecuteTruncateGuts(List *explicit_rels, */ RelationSetNewRelfilenode(rel, rel->rd_rel->relpersistence); + /* update view info */ + if (IS_QD_OR_SINGLENODE()) + SetRelativeMatviewAuxStatus(RelationGetRelid(rel), MV_DATA_STATUS_EXPIRED); + heap_relid = RelationGetRelid(rel); /* @@ -14148,6 +14153,7 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, case OCLASS_TASK: case OCLASS_PROFILE: case OCLASS_PASSWORDHISTORY: + case OCLASS_MATVIEW_AUX: case OCLASS_STORAGE_SERVER: case OCLASS_STORAGE_USER_MAPPING: diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 7877659072b..b418dc106fe 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -36,6 +36,7 @@ #include "access/tableam.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/gp_matview_aux.h" #include "catalog/namespace.h" #include "catalog/partition.h" #include "catalog/pg_database.h" @@ -2581,6 +2582,17 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, else /* Heap vacuum or AO/CO vacuum in specific phase */ table_relation_vacuum(rel, params, vac_strategy); + if (IS_QD_OR_SINGLENODE() && (params->options & VACOPT_FULL)) + { + /* + * Update view data status: + * VACUUM FULL will change the physical pages of table. + * FIXME: for auto vacuum process on segments, it's in utility mode, + * we can't handle it yet. But it's not a problem for SERVERLESS. + */ + SetRelativeMatviewAuxStatus(relid, MV_DATA_STATUS_UP_REORGANIZED); + } + /* Roll back any GUC changes executed by index functions */ AtEOXact_GUC(false, save_nestlevel); diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index a692e4bf3a2..20150bd8d15 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -89,6 +89,7 @@ #include "catalog/pg_tablespace.h" #include "catalog/catalog.h" +#include "catalog/gp_matview_aux.h" #include "catalog/oid_dispatch.h" #include "catalog/pg_directory_table.h" #include "catalog/pg_type.h" @@ -972,6 +973,61 @@ standard_ExecutorRun(QueryDesc *queryDesc, } if ((exec_identity == GP_IGNORE || exec_identity == GP_ROOT_SLICE) && operation != CMD_SELECT) es_processed = mppExecutorWait(queryDesc); + + /* + * Update view info if possible. + * Use the operation and result relation to indentify if + * a table's data is changed. + * This is a proper place for all tables of different AMs. + * We handle INSERT/UPDATE/DELETE here as other operations should + * be handled in utility commands. + * + * Use es_processed to indentify the actual rows we modified + * to avoid case writablte query may not + * change data when success, ex: + * insert into t1 select * from t2; + * When t2 has zero rows, don't need to update view status. + * + * NB: This can't handle well in utility mode, should REFRESH by user + * after that. + */ + if ((GP_ROLE_DISPATCH == Gp_role && es_processed > 0) || + (IS_SINGLENODE() && estate->es_processed > 0)) + { + if (operation == CMD_INSERT || + operation == CMD_UPDATE || + operation == CMD_DELETE) + { + List *rtable =queryDesc->plannedstmt->rtable; + int length = list_length(rtable); + ListCell *lc; + foreach(lc, queryDesc->plannedstmt->resultRelations) + { + int varno = lfirst_int(lc); + + /* Avoid crash in case we don't find a rte. */ + if (varno > length + 1) + { + ereport(WARNING, (errmsg("could not find rte of varno: %u ", varno))); + continue; + } + + RangeTblEntry *rte = rt_fetch(varno, rtable); + switch (operation) + { + case CMD_INSERT: + SetRelativeMatviewAuxStatus(rte->relid, MV_DATA_STATUS_EXPIRED_INSERT_ONLY); + break; + case CMD_UPDATE: + case CMD_DELETE: + SetRelativeMatviewAuxStatus(rte->relid, MV_DATA_STATUS_EXPIRED); + break; + default: + break; + } + } + } + } } PG_CATCH(); { diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index f81c922e3fc..547231c2747 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -93,6 +93,9 @@ #include "catalog/gp_indexing.h" +#include "catalog/gp_matview_aux.h" +#include "catalog/gp_matview_tables.h" + /*--------------------------------------------------------------------------- Adding system caches: @@ -1128,6 +1131,28 @@ static const struct cachedesc cacheinfo[] = { }, 2 }, + {GpMatviewAuxId, /* MVAUXOID */ + GpMatviewAuxMvoidIndexId, + 1, + { + Anum_gp_matview_aux_mvoid, + 0, + 0, + 0 + }, + 32 + }, + {GpMatviewTablesId, /* MVTABLESMVRELOID */ + GpMatviewTablesMvRelIndexId, + 2, + { + Anum_gp_matview_tables_mvoid, + Anum_gp_matview_tables_relid, + 0, + 0 + }, + 128 + }, {UserMappingRelationId, /* USERMAPPINGOID */ UserMappingOidIndexId, 1, diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index c39c0edfa58..cd34fee0ad3 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -152,6 +152,7 @@ typedef enum ObjectClass OCLASS_STORAGE_SERVER, /* gp_storage_server */ OCLASS_STORAGE_USER_MAPPING, /* gp_storage_user_mapping */ OCLASS_EXTPROTOCOL, /* pg_extprotocol */ + OCLASS_MATVIEW_AUX, /* gp_matview_aux */ OCLASS_TASK, /* pg_task */ } ObjectClass; diff --git a/src/include/catalog/gp_matview_aux.h b/src/include/catalog/gp_matview_aux.h new file mode 100644 index 00000000000..a9beaad4244 --- /dev/null +++ b/src/include/catalog/gp_matview_aux.h @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------- + * + * gp_matview_aux.h + * definitions for the gp_matview_aux catalog table + * + * Portions Copyright (c) 2024-Present HashData, Inc. or its affiliates. + * + * + * IDENTIFICATION + * src/include/catalog/gp_matview_aux.h + * + * NOTES + * + *------------------------------------------------------------------------- + */ + +#ifndef GP_MATVIEW_AUX_H +#define GP_MATVIEW_AUX_H + +#include "catalog/genbki.h" +#include "catalog/gp_matview_aux_d.h" + +/* + * Defines for gp_matview_aux + */ +CATALOG(gp_matview_aux,7061,GpMatviewAuxId) BKI_SHARED_RELATION +{ + Oid mvoid; /* materialized view oid */ + NameData mvname; /* materialized view name */ + /* view's data status */ + char datastatus; +} FormData_gp_matview_aux; + + +/* ---------------- + * Form_gp_matview_aux corresponds to a pointer to a tuple with + * the format of gp_matview_aux relation. + * ---------------- + */ +typedef FormData_gp_matview_aux *Form_gp_matview_aux; + +DECLARE_UNIQUE_INDEX(gp_matview_aux_mvoid_index, 7147, on gp_matview_aux using btree(mvoid oid_ops)); +#define GpMatviewAuxMvoidIndexId 7147 + +DECLARE_INDEX(gp_matview_aux_mvname_index, 7148, on gp_matview_aux using btree(mvname name_ops)); +#define GpMatviewAuxMvnameIndexId 7148 + +DECLARE_INDEX(gp_matview_aux_datastatus_index, 7149, on gp_matview_aux using btree(datastatus char_ops)); +#define GpMatviewAuxDatastatusIndexId 7149 + +#define MV_DATA_STATUS_UP_TO_DATE 'u' /* data is up to date */ +#define MV_DATA_STATUS_UP_REORGANIZED 'r' /* data is up to date, but reorganized. VACUUM FULL/CLUSTER */ +#define MV_DATA_STATUS_EXPIRED 'e' /* data is expired */ +#define MV_DATA_STATUS_EXPIRED_INSERT_ONLY 'i' /* expired but has only INSERT operation since latest REFRESH */ + +extern void InsertMatviewAuxEntry(Oid mvoid, const Query *viewQuery, bool skipdata); + +extern void RemoveMatviewAuxEntry(Oid mvoid); + +extern List* GetViewBaseRelids(const Query *viewQuery); + +extern void SetRelativeMatviewAuxStatus(Oid relid, char status); + +extern void SetMatviewAuxStatus(Oid mvoid, char status); + +extern bool MatviewUsableForAppendAgg(Oid mvoid); + +extern bool MatviewIsGeneralyUpToDate(Oid mvoid); + +#endif /* GP_MATVIEW_AUX_H */ diff --git a/src/include/catalog/gp_matview_tables.h b/src/include/catalog/gp_matview_tables.h new file mode 100644 index 00000000000..8592d776033 --- /dev/null +++ b/src/include/catalog/gp_matview_tables.h @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * gp_matview_tables.h + * Definitions for the gp_matview_tables catalog table. + * This is used to record the relations between + * materialized view and base tables. + * + * Portions Copyright (c) 2024-Present HashData, Inc. or its affiliates. + * + * + * IDENTIFICATION + * src/include/catalog/gp_matview_tables.h + * + * NOTES + * This should cooperate with gp_matview_aux. + * + *------------------------------------------------------------------------- + */ + +#ifndef gp_matview_tables_H +#define gp_matview_tables_H + +#include "catalog/genbki.h" +#include "catalog/gp_matview_tables_d.h" + +/* + * Defines for gp_matview_tables + */ +CATALOG(gp_matview_tables,7150,GpMatviewTablesId) BKI_SHARED_RELATION +{ + Oid mvoid; /* materialized view oid */ + Oid relid; /* base table oid */ +} FormData_gp_matview_tables; + + +/* ---------------- + * Form_gp_matview_tables corresponds to a pointer to a tuple with + * the format of gp_matview_tables relation. + * ---------------- + */ +typedef FormData_gp_matview_tables *Form_gp_matview_tables; + +DECLARE_UNIQUE_INDEX(gp_matview_tables_mvoid_relid_index, 7151, on gp_matview_tables using btree(mvoid oid_ops, relid oid_ops)); +#define GpMatviewTablesMvRelIndexId 7151 + +DECLARE_INDEX(gp_matview_tables_relid_index, 7152, on gp_matview_tables using btree(relid oid_ops)); +#define GpMatviewTablesRelIndexId 7152 + +#endif /* gp_matview_tables_H */ diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index 3b493217607..21f0ec0e099 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -75,6 +75,8 @@ extern bool gp_internal_is_singlenode; /* CBDB#69: support single node (no segm #define IS_SINGLENODE() (gp_internal_is_singlenode) #define IS_UTILITY_BUT_NOT_SINGLENODE() (Gp_role == GP_ROLE_UTILITY && !IS_SINGLENODE()) +#define IS_QD_OR_SINGLENODE() ((Gp_role == GP_ROLE_DISPATCH) || IS_SINGLENODE()) + extern GpRoleValue Gp_role; /* GUC var - server operating mode. */ extern char *gp_role_string; /* Use by guc.c as staging area for value. */ diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 62f6b2a46e2..d1653761f43 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -122,6 +122,8 @@ enum SysCacheIdentifier DIRECTORYTABLEREL, STORAGEUSERMAPPINGOID, STORAGEUSERMAPPINGUSERSERVER, + MVAUXOID, + MVTABLESMVRELOID, USERMAPPINGOID, USERMAPPINGUSERSERVER diff --git a/src/test/regress/expected/matview_data.out b/src/test/regress/expected/matview_data.out new file mode 100644 index 00000000000..588692d8ff1 --- /dev/null +++ b/src/test/regress/expected/matview_data.out @@ -0,0 +1,333 @@ +-- disable ORCA +SET optimizer TO off; +create schema matview_data_schema; +set search_path to matview_data_schema; +create table t1(a int, b int); +create table t2(a int, b int); +insert into t1 select i, i+1 from generate_series(1, 5) i; +insert into t1 select i, i+1 from generate_series(1, 3) i; +create materialized view mv0 as select * from t1; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create materialized view mv1 as select a, count(*), sum(b) from t1 group by a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create materialized view mv2 as select * from t2; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- all mv are up to date +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2'); + mvname | datastatus +--------+------------ + mv0 | u + mv1 | u + mv2 | u +(3 rows) + +-- truncate in self transaction +begin; +create table t3(a int, b int); +create materialized view mv3 as select * from t3; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + u +(1 row) + +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + e +(1 row) + +end; +-- trcuncate +refresh materialized view mv3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + u +(1 row) + +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + e +(1 row) + +-- insert and refresh +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +insert into t1 values (1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + i +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + i +(1 row) + +-- insert but no rows changes +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +insert into t1 select * from t3; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +-- update +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +update t1 set a = 10 where a = 1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- delete +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +delete from t1 where a = 10; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- vacuum +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +vacuum t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + r +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + r +(1 row) + +-- insert after vacuum full +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- vacuum full after insert +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + i +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + i +(1 row) + +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- Refresh With No Data +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + u +(1 row) + +refresh materialized view mv2 with no data; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + e +(1 row) + +-- Copy +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + u +(1 row) + +-- 0 rows +COPY t2 from stdin; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + u +(1 row) + +COPY t2 from stdin; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + i +(1 row) + +-- test drop table +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); + mvname | datastatus +--------+------------ + mv3 | e + mv0 | e + mv1 | e + mv2 | i +(4 rows) + +drop materialized view mv2; +drop table t1 cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to materialized view mv0 +drop cascades to materialized view mv1 +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); + mvname | datastatus +--------+------------ + mv3 | e +(1 row) + +drop schema matview_data_schema cascade; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table t2 +drop cascades to table t3 +drop cascades to materialized view mv3 +reset enable_answer_query_using_materialized_views; +reset optimizer; diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out index 7e1df68863b..8bd8db7e70b 100644 --- a/src/test/regress/expected/misc_sanity.out +++ b/src/test/regress/expected/misc_sanity.out @@ -152,6 +152,8 @@ ORDER BY 1; gp_distribution_policy gp_fastsequence gp_id + gp_matview_aux + gp_matview_tables gp_partition_template gp_version_at_initdb gp_warehouse @@ -173,7 +175,7 @@ ORDER BY 1; pg_stat_last_operation pg_stat_last_shoperation pg_type_encoding -(25 rows) +(27 rows) -- system catalog unique indexes not wrapped in a constraint -- (There should be none.) diff --git a/src/test/regress/expected/misc_sanity_external_fts.out b/src/test/regress/expected/misc_sanity_external_fts.out index 09f55a8faca..1c0ac1b7ccb 100644 --- a/src/test/regress/expected/misc_sanity_external_fts.out +++ b/src/test/regress/expected/misc_sanity_external_fts.out @@ -151,6 +151,8 @@ ORDER BY 1; gp_distribution_policy gp_fastsequence gp_id + gp_matview_aux + gp_matview_tables gp_partition_template gp_version_at_initdb gp_warehouse @@ -172,7 +174,7 @@ ORDER BY 1; pg_stat_last_operation pg_stat_last_shoperation pg_type_encoding -(25 rows) +(27 rows) -- system catalog unique indexes not wrapped in a constraint -- (There should be none.) diff --git a/src/test/regress/expected/pg_ext_aux.out b/src/test/regress/expected/pg_ext_aux.out index 9e6f21fa7b7..7e7138ab54d 100644 --- a/src/test/regress/expected/pg_ext_aux.out +++ b/src/test/regress/expected/pg_ext_aux.out @@ -59,7 +59,7 @@ insert into pg_ext_aux.extaux_t values(1,'hello'); ERROR: permission denied: "extaux_t" is a system catalog -- fail: should not allowed to refresh by user refresh materialized view pg_ext_aux.extaux_mv; -ERROR: cannot swap toast files by links for system catalogs (cluster.c:1424) +ERROR: cannot swap toast files by links for system catalogs (cluster.c:XXX) -- fail: should not allow to be dropped by user drop view pg_ext_aux.extaux_v; ERROR: permission denied: "extaux_v" is a system catalog diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 577ed66a383..a9892846caf 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -315,6 +315,9 @@ test: offload_entry_to_qe # Tests of Answer Query Using Materialized Views. test: aqumv +# Tests of materialized view data catalog maintenance +test: matview_data + # test access method with encoding options test: am_encoding diff --git a/src/test/regress/init_file b/src/test/regress/init_file index 65e735a8e00..8c7237d1bb4 100644 --- a/src/test/regress/init_file +++ b/src/test/regress/init_file @@ -127,6 +127,9 @@ s/ERROR: could not find hash function for hash operator.*/ERROR: could not fin m/ERROR: could not devise a plan.*/ s/ERROR: could not devise a plan.*/ERROR: could not devise a plan (cdbpath.c:XXX)/ +m/ERROR: cannot swap toast files by links for system catalogs.*/ +s/ERROR: cannot swap toast files by links for system catalogs.*/ERROR: cannot swap toast files by links for system catalogs (cluster.c:XXX)/ + m/nodename nor servname provided, or not known/ s/nodename nor servname provided, or not known/Name or service not known/ diff --git a/src/test/regress/sql/matview_data.sql b/src/test/regress/sql/matview_data.sql new file mode 100644 index 00000000000..24a701ef584 --- /dev/null +++ b/src/test/regress/sql/matview_data.sql @@ -0,0 +1,119 @@ +-- disable ORCA +SET optimizer TO off; +create schema matview_data_schema; +set search_path to matview_data_schema; + +create table t1(a int, b int); +create table t2(a int, b int); +insert into t1 select i, i+1 from generate_series(1, 5) i; +insert into t1 select i, i+1 from generate_series(1, 3) i; +create materialized view mv0 as select * from t1; +create materialized view mv1 as select a, count(*), sum(b) from t1 group by a; +create materialized view mv2 as select * from t2; +-- all mv are up to date +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2'); + +-- truncate in self transaction +begin; +create table t3(a int, b int); +create materialized view mv3 as select * from t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; +end; + +-- trcuncate +refresh materialized view mv3; +select datastatus from gp_matview_aux where mvname = 'mv3'; +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + +-- insert and refresh +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +insert into t1 values (1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- insert but no rows changes +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +insert into t1 select * from t3; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- update +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +update t1 set a = 10 where a = 1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- delete +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +delete from t1 where a = 10; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- vacuum +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +vacuum t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +-- insert after vacuum full +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +-- vacuum full after insert +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- Refresh With No Data +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; +refresh materialized view mv2 with no data; +select datastatus from gp_matview_aux where mvname = 'mv2'; + +-- Copy +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; +-- 0 rows +COPY t2 from stdin; +\. +select datastatus from gp_matview_aux where mvname = 'mv2'; + +COPY t2 from stdin; +1 1 +\. +select datastatus from gp_matview_aux where mvname = 'mv2'; + +-- test drop table +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); +drop materialized view mv2; +drop table t1 cascade; +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); + +drop schema matview_data_schema cascade; +reset enable_answer_query_using_materialized_views; +reset optimizer; diff --git a/src/test/singlenode_regress/expected/matview_data.out b/src/test/singlenode_regress/expected/matview_data.out new file mode 100644 index 00000000000..588692d8ff1 --- /dev/null +++ b/src/test/singlenode_regress/expected/matview_data.out @@ -0,0 +1,333 @@ +-- disable ORCA +SET optimizer TO off; +create schema matview_data_schema; +set search_path to matview_data_schema; +create table t1(a int, b int); +create table t2(a int, b int); +insert into t1 select i, i+1 from generate_series(1, 5) i; +insert into t1 select i, i+1 from generate_series(1, 3) i; +create materialized view mv0 as select * from t1; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create materialized view mv1 as select a, count(*), sum(b) from t1 group by a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create materialized view mv2 as select * from t2; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- all mv are up to date +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2'); + mvname | datastatus +--------+------------ + mv0 | u + mv1 | u + mv2 | u +(3 rows) + +-- truncate in self transaction +begin; +create table t3(a int, b int); +create materialized view mv3 as select * from t3; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Cloudberry Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + u +(1 row) + +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + e +(1 row) + +end; +-- trcuncate +refresh materialized view mv3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + u +(1 row) + +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + datastatus +------------ + e +(1 row) + +-- insert and refresh +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +insert into t1 values (1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + i +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + i +(1 row) + +-- insert but no rows changes +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +insert into t1 select * from t3; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +-- update +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +update t1 set a = 10 where a = 1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- delete +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +delete from t1 where a = 10; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- vacuum +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +vacuum t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + r +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + r +(1 row) + +-- insert after vacuum full +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- vacuum full after insert +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + u +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + u +(1 row) + +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + i +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + i +(1 row) + +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; + datastatus +------------ + e +(1 row) + +select datastatus from gp_matview_aux where mvname = 'mv1'; + datastatus +------------ + e +(1 row) + +-- Refresh With No Data +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + u +(1 row) + +refresh materialized view mv2 with no data; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + e +(1 row) + +-- Copy +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + u +(1 row) + +-- 0 rows +COPY t2 from stdin; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + u +(1 row) + +COPY t2 from stdin; +select datastatus from gp_matview_aux where mvname = 'mv2'; + datastatus +------------ + i +(1 row) + +-- test drop table +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); + mvname | datastatus +--------+------------ + mv3 | e + mv0 | e + mv1 | e + mv2 | i +(4 rows) + +drop materialized view mv2; +drop table t1 cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to materialized view mv0 +drop cascades to materialized view mv1 +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); + mvname | datastatus +--------+------------ + mv3 | e +(1 row) + +drop schema matview_data_schema cascade; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table t2 +drop cascades to table t3 +drop cascades to materialized view mv3 +reset enable_answer_query_using_materialized_views; +reset optimizer; diff --git a/src/test/singlenode_regress/expected/misc_sanity.out b/src/test/singlenode_regress/expected/misc_sanity.out index 7e1df68863b..8bd8db7e70b 100644 --- a/src/test/singlenode_regress/expected/misc_sanity.out +++ b/src/test/singlenode_regress/expected/misc_sanity.out @@ -152,6 +152,8 @@ ORDER BY 1; gp_distribution_policy gp_fastsequence gp_id + gp_matview_aux + gp_matview_tables gp_partition_template gp_version_at_initdb gp_warehouse @@ -173,7 +175,7 @@ ORDER BY 1; pg_stat_last_operation pg_stat_last_shoperation pg_type_encoding -(25 rows) +(27 rows) -- system catalog unique indexes not wrapped in a constraint -- (There should be none.) diff --git a/src/test/singlenode_regress/greenplum_schedule b/src/test/singlenode_regress/greenplum_schedule index b682d230aa1..7883d89de0b 100755 --- a/src/test/singlenode_regress/greenplum_schedule +++ b/src/test/singlenode_regress/greenplum_schedule @@ -310,6 +310,8 @@ test: AOCO_Compression AORO_Compression # check correct error message when create extension error on segment # test: create_extension_fail +test: matview_data + # CBDB specific tests test: singlenode_compatibility_cbdb # end of tests diff --git a/src/test/singlenode_regress/sql/matview_data.sql b/src/test/singlenode_regress/sql/matview_data.sql new file mode 100644 index 00000000000..24a701ef584 --- /dev/null +++ b/src/test/singlenode_regress/sql/matview_data.sql @@ -0,0 +1,119 @@ +-- disable ORCA +SET optimizer TO off; +create schema matview_data_schema; +set search_path to matview_data_schema; + +create table t1(a int, b int); +create table t2(a int, b int); +insert into t1 select i, i+1 from generate_series(1, 5) i; +insert into t1 select i, i+1 from generate_series(1, 3) i; +create materialized view mv0 as select * from t1; +create materialized view mv1 as select a, count(*), sum(b) from t1 group by a; +create materialized view mv2 as select * from t2; +-- all mv are up to date +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2'); + +-- truncate in self transaction +begin; +create table t3(a int, b int); +create materialized view mv3 as select * from t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; +end; + +-- trcuncate +refresh materialized view mv3; +select datastatus from gp_matview_aux where mvname = 'mv3'; +truncate t3; +select datastatus from gp_matview_aux where mvname = 'mv3'; + +-- insert and refresh +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +insert into t1 values (1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- insert but no rows changes +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +insert into t1 select * from t3; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- update +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +update t1 set a = 10 where a = 1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- delete +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +delete from t1 where a = 10; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- vacuum +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +vacuum t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +-- insert after vacuum full +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +-- vacuum full after insert +refresh materialized view mv0; +refresh materialized view mv1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +insert into t1 values(1, 2); +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; +vacuum full t1; +select datastatus from gp_matview_aux where mvname = 'mv0'; +select datastatus from gp_matview_aux where mvname = 'mv1'; + +-- Refresh With No Data +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; +refresh materialized view mv2 with no data; +select datastatus from gp_matview_aux where mvname = 'mv2'; + +-- Copy +refresh materialized view mv2; +select datastatus from gp_matview_aux where mvname = 'mv2'; +-- 0 rows +COPY t2 from stdin; +\. +select datastatus from gp_matview_aux where mvname = 'mv2'; + +COPY t2 from stdin; +1 1 +\. +select datastatus from gp_matview_aux where mvname = 'mv2'; + +-- test drop table +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); +drop materialized view mv2; +drop table t1 cascade; +select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); + +drop schema matview_data_schema cascade; +reset enable_answer_query_using_materialized_views; +reset optimizer;