From 6eb91d76fadc5104b7a17c0c0e4679fcfb37033a Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Fri, 29 Mar 2024 18:15:24 +0000 Subject: [PATCH 1/3] init --- ydb/library/yql/ast/yql_expr.cpp | 8 ++++ ydb/library/yql/ast/yql_expr.h | 42 ++++++++++++----- .../yql/core/common_opt/yql_co_pgselect.cpp | 4 +- ydb/library/yql/core/type_ann/type_ann_pg.cpp | 38 ++++++++------- ydb/library/yql/parser/pg_catalog/catalog.cpp | 21 +++++++-- ydb/library/yql/parser/pg_catalog/catalog.h | 2 +- .../yql/parser/pg_wrapper/comp_factory.cpp | 46 +++++++++++++++++++ .../provider/yql_pg_datasource_type_ann.cpp | 10 ++++ ydb/library/yql/sql/pg/pg_sql.cpp | 6 +++ ydb/library/yql/sql/pg/pg_sql_ut.cpp | 9 ++++ .../sql/dq_file/part11/canondata/result.json | 22 +++++++++ .../hybrid_file/part6/canondata/result.json | 14 ++++++ .../tests/sql/sql2yql/canondata/result.json | 7 +++ .../sql/suites/pg_catalog/system_columns.sql | 3 ++ .../part11/canondata/result.json | 21 +++++++++ 15 files changed, 218 insertions(+), 35 deletions(-) create mode 100644 ydb/library/yql/tests/sql/suites/pg_catalog/system_columns.sql diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 6c6051cf134c..827e73b9967a 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -3029,6 +3029,14 @@ bool TItemExprType::Validate(TPositionHandle position, TExprContext& ctx) const return Validate(ctx.GetPosition(position), ctx); } +const TItemExprType* TItemExprType::GetCleanItem(bool isVirtual, TExprContext& ctx) const { + if (!isVirtual) { + return this; + } + + return ctx.MakeType(Name.SubStr(YqlVirtualPrefix.size()), ItemType); +} + bool TMultiExprType::Validate(TPosition position, TExprContext& ctx) const { if (Items.size() > Max()) { ctx.AddError(TIssue(position, TStringBuilder() << "Too many elements: " << Items.size())); diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index 24671c76c16b..2ba6db361639 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -85,6 +85,7 @@ class TBlockExprType; class TScalarExprType; const size_t DefaultMistypeDistance = 3; +const TString YqlVirtualPrefix = "_yql_virtual_"; extern const TStringBuf ZeroString; @@ -464,6 +465,10 @@ class TItemExprType : public TTypeAnnotationNode { return Name; } + TStringBuf GetCleanName(bool isVirtual) const { + return isVirtual ? Name.SubStr(YqlVirtualPrefix.size()) : Name; + } + const TTypeAnnotationNode* GetItemType() const { return ItemType; } @@ -472,6 +477,8 @@ class TItemExprType : public TTypeAnnotationNode { return GetName() == other.GetName() && GetItemType() == other.GetItemType(); } + const TItemExprType* GetCleanItem(bool isVirtual, TExprContext& ctx) const; + private: const TStringBuf Name; const TTypeAnnotationNode* ItemType; @@ -532,24 +539,35 @@ class TStructExprType : public TTypeAnnotationNode { return it - Items.begin(); } - TMaybe FindItemI(const TStringBuf& name) const { - auto strict = FindItem(name); - if (strict) { - return strict; - } + TMaybe FindItemI(const TStringBuf& name, bool* isVirtual) const { + for (ui32 v = 0; v < 2; ++v) { + if (isVirtual) { + *isVirtual = v > 0; + } - TMaybe ret; - for (ui32 i = 0; i < Items.size(); ++i) { - if (AsciiEqualsIgnoreCase(name, Items[i]->GetName())) { - if (ret) { - return Nothing(); + auto nameToSearch = (v ? YqlVirtualPrefix : "") + name; + auto strict = FindItem(nameToSearch); + if (strict) { + return strict; + } + + TMaybe ret; + for (ui32 i = 0; i < Items.size(); ++i) { + if (AsciiEqualsIgnoreCase(nameToSearch, Items[i]->GetName())) { + if (ret) { + return Nothing(); + } + + ret = i; } + } - ret = i; + if (ret) { + return ret; } } - return ret; + return Nothing(); } const TTypeAnnotationNode* FindItemType(const TStringBuf& name) const { diff --git a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp index 74e9754fa39d..422c9ebd0658 100644 --- a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp @@ -1079,7 +1079,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE } else { auto type = read.GetTypeAnn()-> Cast()->GetItemType()->Cast(); - auto pos = type->FindItem(column); + auto pos = type->FindItemI(column, nullptr); foundColumn = pos.Defined(); } } @@ -1104,7 +1104,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE auto type = input->Tail().GetTypeAnn()-> Cast()->GetType()->Cast(); - auto pos = type->FindItem(column); + auto pos = type->FindItemI(column, nullptr); foundColumn = pos.Defined(); if (foundColumn) { x.second.first = inputIndex; diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.cpp b/ydb/library/yql/core/type_ann/type_ann_pg.cpp index b58e142aa742..9492ba9c5e48 100644 --- a/ydb/library/yql/core/type_ann/type_ann_pg.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_pg.cpp @@ -151,7 +151,7 @@ IGraphTransformer::TStatus InferPgCommonType(TPositionHandle pos, const TExprNod { size_t j = 0; for (const auto& col : *childColumnOrder) { - auto itemIdx = structType->FindItemI(col); + auto itemIdx = structType->FindItemI(col, nullptr); YQL_ENSURE(itemIdx); const auto* type = structType->GetItems()[*itemIdx]->GetItemType(); @@ -1725,7 +1725,8 @@ bool ScanColumns(TExprNode::TPtr root, TInputs& inputs, const THashSet& } } - auto pos = x.Type->FindItemI(node->Tail().Content()); + bool isVirtual; + auto pos = x.Type->FindItemI(node->Tail().Content(), &isVirtual); if (pos) { foundAlias = x.Alias; ++matches; @@ -1737,7 +1738,8 @@ bool ScanColumns(TExprNode::TPtr root, TInputs& inputs, const THashSet& } if (x.Priority == TInput::External) { - x.UsedExternalColumns.insert(TString(x.Type->GetItems()[*pos]->GetName())); + auto name = TString(x.Type->GetItems()[*pos]->GetCleanName(isVirtual)); + x.UsedExternalColumns.insert(name); } } } @@ -1966,17 +1968,18 @@ void AddColumns(const TInputs& inputs, const bool* hasStar, const THashSetFindItemI(ref); + bool isVirtual; + auto pos = x.Type->FindItemI(ref, &isVirtual); if (pos) { auto item = x.Type->GetItems()[*pos]; - TString lcase = to_lower(TString(item->GetName())); + TString lcase = to_lower(TString(item->GetCleanName(isVirtual))); if (auto it = usedInUsing.find(lcase); it != usedInUsing.end() && !present.contains(lcase)) { items.push_back(ctx.MakeType(it->second, item->GetItemType())); present.emplace(lcase); } item = AddAlias(x.Alias, item, ctx); items.push_back(item); - usedRefs.insert(TString(item->GetName())); + usedRefs.insert(TString(item->GetCleanName(isVirtual))); } } @@ -1986,7 +1989,7 @@ void AddColumns(const TInputs& inputs, const bool* hasStar, const THashSetfind(x.Alias)->second) { - auto pos = x.Type->FindItemI(ref); + auto pos = x.Type->FindItemI(ref, nullptr); if (pos) { auto item = x.Type->GetItems()[*pos]; item = AddAlias(x.Alias, item, ctx); @@ -2110,7 +2113,7 @@ IGraphTransformer::TStatus RebuildLambdaColumns(const TExprNode::TPtr& root, con } } - auto pos = x.Type->FindItemI(node->Tail().Content()); + auto pos = x.Type->FindItemI(node->Tail().Content(), nullptr); if (pos) { return ctx.Expr.Builder(node->Pos()) .Callable("Member") @@ -3119,7 +3122,7 @@ bool GatherExtraSortColumns(const TExprNode& data, const TInputs& inputs, TExprN continue; } - auto pos = x.Type->FindItemI(column); + auto pos = x.Type->FindItemI(column, nullptr); if (pos) { index = inputIndex; break; @@ -3791,7 +3794,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN TVector newStructItems; TColumnOrder newOrder; for (ui32 i = 0; i < p->Child(2)->ChildrenSize(); ++i) { - auto pos = inputStructType->FindItemI((*columnOrder)[i]); + auto pos = inputStructType->FindItemI((*columnOrder)[i], nullptr); YQL_ENSURE(pos); auto type = inputStructType->GetItems()[*pos]->GetItemType(); newOrder.push_back(TString(p->Child(2)->Child(i)->Content())); @@ -4138,11 +4141,12 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN } else { int matchCount = 0; for (size_t j = 0; j <= groupInputs.size() - 2; ++j) { - auto pos = groupInputs[j].Type->FindItemI(name); + bool isVirtual; + auto pos = groupInputs[j].Type->FindItemI(name, &isVirtual); if (!pos) { continue; } - lrNames[0] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[j].Alias, groupInputs[j].Type->GetItems()[*pos]->GetName())); + lrNames[0] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[j].Alias, groupInputs[j].Type->GetItems()[*pos]->GetCleanName(isVirtual))); ++matchCount; } if (!matchCount) { @@ -4154,13 +4158,14 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN return IGraphTransformer::TStatus::Error; } } - auto pos = rightInput.Type->FindItemI(name); + bool isVirtual; + auto pos = rightInput.Type->FindItemI(name, &isVirtual); usedInUsingBefore.emplace(lcase); if (!pos) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Can't find column: " << name)); return IGraphTransformer::TStatus::Error; } - lrNames[1] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(rightInput.Alias, rightInput.Type->GetItems()[*pos]->GetName())); + lrNames[1] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(rightInput.Alias, rightInput.Type->GetItems()[*pos]->GetCleanName(isVirtual))); nodes[colIdx] = ctx.Expr.NewList(inp->Pos(), std::move(lrNames)); } TExprNode::TListType newJoin(4); @@ -4616,9 +4621,10 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN const auto type = data.Child(j)->Tail().GetTypeAnn()->Cast()-> GetType()->Cast(); for (const auto& col : x.UsedExternalColumns) { - auto pos = type->FindItemI(col); + bool isVirtual; + auto pos = type->FindItemI(col, &isVirtual); YQL_ENSURE(pos); - items.push_back(type->GetItems()[*pos]); + items.push_back(type->GetItems()[*pos]->GetCleanItem(isVirtual, ctx.Expr)); } auto effectiveType = ctx.Expr.MakeType(items); diff --git a/ydb/library/yql/parser/pg_catalog/catalog.cpp b/ydb/library/yql/parser/pg_catalog/catalog.cpp index e0fb3d3e74d3..cacdd231542a 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.cpp +++ b/ydb/library/yql/parser/pg_catalog/catalog.cpp @@ -1604,7 +1604,7 @@ struct TCatalog { "lo_close", "lo_unlink" }), - StaticTables({ + AllStaticTables({ #include "pg_class.generated.h" }), AllStaticColumns({ @@ -1612,9 +1612,10 @@ struct TCatalog { }) { THashSet usedTableOids; - for (const auto& t : StaticTables) { + for (const auto& t : AllStaticTables) { StaticColumns.insert(std::make_pair(t, TVector())); Y_ENSURE(usedTableOids.insert(t.Oid).first); + StaticTables.insert(std::make_pair(TTableInfoKey(t), t)); } for (const auto& c: AllStaticColumns) { @@ -1834,8 +1835,9 @@ struct TCatalog { THashMap> AggregationsByName; THashSet ProhibitedProcs; - TVector StaticTables; + TVector AllStaticTables; TVector AllStaticColumns; + THashMap StaticTables; THashMap> StaticColumns; }; @@ -3079,7 +3081,7 @@ void EnumLanguages(std::function f) { const TVector& GetStaticTables() { const auto& catalog = TCatalog::Instance(); - return catalog.StaticTables; + return catalog.AllStaticTables; } const THashMap>& GetStaticColumns() { @@ -3087,4 +3089,15 @@ const THashMap>& GetStaticColumns() { return catalog.StaticColumns; } +const TTableInfo& LookupStaticTable(const TTableInfoKey& tableKey) { + const auto& catalog = TCatalog::Instance(); + auto tablePtr = catalog.StaticTables.FindPtr(tableKey); + if (!tablePtr) { + throw yexception() << "No such table: " << tableKey.Schema << "." << tableKey.Name; + } + + return *tablePtr; +} + + } diff --git a/ydb/library/yql/parser/pg_catalog/catalog.h b/ydb/library/yql/parser/pg_catalog/catalog.h index f35f2113654f..cd76abcd7b43 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.h +++ b/ydb/library/yql/parser/pg_catalog/catalog.h @@ -353,8 +353,8 @@ struct TColumnInfo { }; const TVector& GetStaticTables(); +const TTableInfo& LookupStaticTable(const TTableInfoKey& tableKey); const THashMap>& GetStaticColumns(); - } template <> diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index bba184358b56..9b5d21665398 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -502,9 +502,33 @@ class TPgTableContent : public TMutableComputationNode { } } + class TSystemColumnFiller { + public: + TSystemColumnFiller(TStructType* itemType, const TString& cluster, const TString& table) { + const auto& info = NPg::LookupStaticTable(NPg::TTableInfoKey(cluster, table)); + TableOid = info.Oid; + if (info.Kind != NPg::ERelKind::Relation) { + return; + } + + TableOidPos = itemType->FindMemberIndex("_yql_virtual_tableoid"); + } + + void Fill(NUdf::TUnboxedValue* items) { + if (TableOidPos) { + items[*TableOidPos] = ScalarDatumToPod(Int32GetDatum(TableOid)); + } + } + + private: + ui32 TableOid = 0; + TMaybe TableOidPos; + }; + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& compCtx) const { TUnboxedValueVector rows; if (Cluster_ == "pg_catalog") { + TSystemColumnFiller sysFiller(ItemType_, TString(Cluster_), TString(Table_)); if (Table_ == "pg_type") { NPg::EnumTypes([&](ui32 oid, const NPg::TTypeDesc& desc) { NUdf::TUnboxedValue* items; @@ -513,8 +537,10 @@ class TPgTableContent : public TMutableComputationNode { if (PgTypeFillers_[i]) { items[i] = PgTypeFillers_[i](desc); } + } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_database") { @@ -527,6 +553,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } } else if (Table_ == "pg_tablespace") { @@ -539,6 +566,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } } else if (Table_ == "pg_shdescription") { @@ -551,6 +579,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } } else if (Table_ == "pg_stat_gssapi") { @@ -562,6 +591,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } else if (Table_ == "pg_namespace") { NPg::EnumNamespace([&](ui32 oid, const NPg::TNamespaceDesc& desc) { @@ -573,6 +603,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_am") { @@ -585,6 +616,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_description") { @@ -601,6 +633,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); @@ -616,6 +649,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); @@ -631,6 +665,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); @@ -647,6 +682,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); @@ -663,6 +699,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); @@ -679,6 +716,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_tables") { @@ -692,6 +730,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } } else if (Table_ == "pg_roles") { @@ -703,6 +742,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } else if (Table_ == "pg_stat_database") { for (ui32 index = 0; index <= 1; ++index) { @@ -714,6 +754,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } } else if (Table_ == "pg_class") { @@ -740,6 +781,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); } } else if (Table_ == "pg_proc") { @@ -752,6 +794,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_operator") { @@ -764,6 +807,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_aggregate") { @@ -776,6 +820,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } else if (Table_ == "pg_language") { @@ -788,6 +833,7 @@ class TPgTableContent : public TMutableComputationNode { } } + sysFiller.Fill(items); rows.emplace_back(row); }); } diff --git a/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp b/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp index aac3069c9b8a..cbded798a6eb 100644 --- a/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp @@ -117,6 +117,16 @@ class TPgDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { AddColumn(items, ctx, c.Name, c.UdtType); } + const auto relKind = NPg::LookupStaticTable(NPg::TTableInfoKey{ cluster, TString(tableName) }).Kind; + if (relKind == NPg::ERelKind::Relation) { + AddColumn(items, ctx, "_yql_virtual_tableoid", "oid"); + AddColumn(items, ctx, "_yql_virtual_xmin", "xid"); + AddColumn(items, ctx, "_yql_virtual_cmin", "cid"); + AddColumn(items, ctx, "_yql_virtual_xmax", "xid"); + AddColumn(items, ctx, "_yql_virtual_cmax", "cid"); + AddColumn(items, ctx, "_yql_virtual_ctid", "tid"); + } + TVector columnOrder; for (const auto& item : items) { columnOrder.emplace_back(TString(item->GetName())); diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp index 410976f83808..63a660e8d301 100644 --- a/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/ydb/library/yql/sql/pg/pg_sql.cpp @@ -53,6 +53,8 @@ namespace NSQLTranslationPG { using namespace NYql; +static const THashSet SystemColumns = { "tableoid", "xmin", "cmin", "xmax", "cmax", "ctid" }; + template const T* CastNode(const void* nodeptr, int tag) { Y_ENSURE(nodeTag(nodeptr) == tag); @@ -1790,6 +1792,10 @@ class TConverter : public IPGParseEvents { bool AddColumn(TCreateTableCtx& ctx, const ColumnDef* node) { TColumnInfo cinfo{.Name = node->colname}; + if (SystemColumns.contains(to_lower(cinfo.Name))) { + AddError(TStringBuilder() << "system column can't be used: " << node->colname); + return false; + } if (node->constraints) { for (int i = 0; i < ListLength(node->constraints); ++i) { diff --git a/ydb/library/yql/sql/pg/pg_sql_ut.cpp b/ydb/library/yql/sql/pg/pg_sql_ut.cpp index 77fa0b91ec2d..fec158773aa1 100644 --- a/ydb/library/yql/sql/pg/pg_sql_ut.cpp +++ b/ydb/library/yql/sql/pg/pg_sql_ut.cpp @@ -111,6 +111,15 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { UNIT_ASSERT_STRINGS_EQUAL(res.Root->ToString(), expectedAst.Root->ToString()); } + Y_UNIT_TEST(CreateTableStmt_SystemColumns) { + auto res = PgSqlToYql("CREATE TABLE t(XMIN int)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_EQUAL(res.Issues.Size(), 1); + + auto issue = *(res.Issues.begin()); + UNIT_ASSERT(issue.GetMessage().find("system column") != TString::npos); + } + Y_UNIT_TEST(CreateTableStmt_NotNull) { auto res = PgSqlToYql("CREATE TABLE t (a int NOT NULL, b text)"); UNIT_ASSERT(res.Root); diff --git a/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json index 91b8d03f2f91..b7219f006c14 100644 --- a/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json @@ -2146,6 +2146,28 @@ } ], "test.test[pg_catalog-pg_roles-default.txt-Results]": [], + "test.test[pg_catalog-system_columns-default.txt-Analyze]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1936273/364381182da99c268c68cfaa8dd7a6e282153b85/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Analyze_/plan.txt" + } + ], + "test.test[pg_catalog-system_columns-default.txt-Debug]": [ + { + "checksum": "1f56c4687e571573ddb2674e1dbad1b4", + "size": 1138, + "uri": "https://{canondata_backend}/1936273/364381182da99c268c68cfaa8dd7a6e282153b85/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg_catalog-system_columns-default.txt-Plan]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1936273/364381182da99c268c68cfaa8dd7a6e282153b85/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg_catalog-system_columns-default.txt-Results]": [], "test.test[pg_catalog-user-default.txt-Analyze]": [ { "checksum": "a3b64a2cf9903b3868a2dd88a18fc46e", diff --git a/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json index 89361f9deb9a..ffdad675fce5 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json @@ -2141,6 +2141,20 @@ "uri": "https://{canondata_backend}/1031349/201452dd8c883b2adcbf46cb075c912d25efe67e/resource.tar.gz#test.test_pg-wide_sort--Plan_/plan.txt" } ], + "test.test[pg_catalog-system_columns-default.txt-Debug]": [ + { + "checksum": "5e20f177453d0eb5836f1216766a7664", + "size": 1137, + "uri": "https://{canondata_backend}/1031349/ac3fdb59ac5555bb04f369156daa910ae69f4f8f/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg_catalog-system_columns-default.txt-Plan]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1031349/ac3fdb59ac5555bb04f369156daa910ae69f4f8f/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Plan_/plan.txt" + } + ], "test.test[produce-process_streaming_inline_bash-default.txt-Debug]": [ { "checksum": "e0416b905170678f107c878710fb2df0", diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index 7ad1aaf5a8c2..f87005ab983a 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -13985,6 +13985,13 @@ "uri": "https://{canondata_backend}/1942173/cb2fdcf9620011c608e9c76afbfe1a17bdd9d6f4/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_type_syntax_yql_/sql.yql" } ], + "test_sql2yql.test[pg_catalog-system_columns]": [ + { + "checksum": "de61a6966af369b0cff9d3b65fb48b99", + "size": 1136, + "uri": "https://{canondata_backend}/1936997/664bdce7dcb090ac953e59b66dae38f4a1fa9bd2/resource.tar.gz#test_sql2yql.test_pg_catalog-system_columns_/sql.yql" + } + ], "test_sql2yql.test[pg_catalog-table_constraints]": [ { "checksum": "036fce68a853ccc7236f6ce01754eaf8", diff --git a/ydb/library/yql/tests/sql/suites/pg_catalog/system_columns.sql b/ydb/library/yql/tests/sql/suites/pg_catalog/system_columns.sql new file mode 100644 index 000000000000..d3388c8cade4 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg_catalog/system_columns.sql @@ -0,0 +1,3 @@ +--!syntax_pg +select oid,tableoid,xmin,cmin,xmax,cmax,ctid from pg_type where typname = 'text'; + diff --git a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json index 7d85691d69a2..34f794fbde64 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json @@ -1924,6 +1924,27 @@ "uri": "https://{canondata_backend}/1942671/ba1b384ae189e43a41610dd3346b04832beabd4c/resource.tar.gz#test.test_pg_catalog-pg_roles-default.txt-Results_/results.txt" } ], + "test.test[pg_catalog-system_columns-default.txt-Debug]": [ + { + "checksum": "4f040bd049ad2202847565c2a3e9735a", + "size": 1077, + "uri": "https://{canondata_backend}/1942278/8a7a88cead6b1c998bd4f3a32c8b3cbf62b2eba4/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg_catalog-system_columns-default.txt-Plan]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1942278/8a7a88cead6b1c998bd4f3a32c8b3cbf62b2eba4/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg_catalog-system_columns-default.txt-Results]": [ + { + "checksum": "a3752ec973fb57518ec89dec4973a090", + "size": 2371, + "uri": "https://{canondata_backend}/1942278/8a7a88cead6b1c998bd4f3a32c8b3cbf62b2eba4/resource.tar.gz#test.test_pg_catalog-system_columns-default.txt-Results_/results.txt" + } + ], "test.test[pg_catalog-user-default.txt-Debug]": [ { "checksum": "7ab5adcfb187ee635f09477cf2928c4f", From eac0e8c7855c4f9d56a162c2df887ef330955548 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Fri, 29 Mar 2024 18:41:07 +0000 Subject: [PATCH 2/3] tests --- .../pg/provider/yql_pg_datasource_type_ann.cpp | 16 ++++++++++------ .../sql/dq_file/part13/canondata/result.json | 6 +++--- .../sql/dq_file/part17/canondata/result.json | 6 +++--- .../sql/hybrid_file/part0/canondata/result.json | 12 ++++++------ .../yt_native_file/part13/canondata/result.json | 6 +++--- .../yt_native_file/part17/canondata/result.json | 6 +++--- 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp b/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp index cbded798a6eb..efed578e1c2c 100644 --- a/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp @@ -119,12 +119,12 @@ class TPgDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { const auto relKind = NPg::LookupStaticTable(NPg::TTableInfoKey{ cluster, TString(tableName) }).Kind; if (relKind == NPg::ERelKind::Relation) { - AddColumn(items, ctx, "_yql_virtual_tableoid", "oid"); - AddColumn(items, ctx, "_yql_virtual_xmin", "xid"); - AddColumn(items, ctx, "_yql_virtual_cmin", "cid"); - AddColumn(items, ctx, "_yql_virtual_xmax", "xid"); - AddColumn(items, ctx, "_yql_virtual_cmax", "cid"); - AddColumn(items, ctx, "_yql_virtual_ctid", "tid"); + AddSystemColumn(items, ctx, "tableoid", "oid"); + AddSystemColumn(items, ctx, "xmin", "xid"); + AddSystemColumn(items, ctx, "cmin", "cid"); + AddSystemColumn(items, ctx, "xmax", "xid"); + AddSystemColumn(items, ctx, "cmax", "cid"); + AddSystemColumn(items, ctx, "ctid", "tid"); } TVector columnOrder; @@ -157,6 +157,10 @@ class TPgDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { items.push_back(ctx.MakeType(name, ctx.MakeType(NPg::LookupType(type).TypeId))); } + void AddSystemColumn(TVector& items, TExprContext& ctx, const TString& name, const TString& type) { + AddColumn(items, ctx, YqlVirtualPrefix + name, type); + } + private: TPgState::TPtr State_; }; diff --git a/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json index d0386e8d3a8f..83cfa4052a5a 100644 --- a/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json @@ -2232,9 +2232,9 @@ ], "test.test[pg_catalog-pg_shdescription-default.txt-Debug]": [ { - "checksum": "70598e00cd83a71031e60339828d4f06", - "size": 526, - "uri": "https://{canondata_backend}/1936997/93899b3de50fae3f9677baacc98094a7a629590a/resource.tar.gz#test.test_pg_catalog-pg_shdescription-default.txt-Debug_/opt.yql_patched" + "checksum": "71da7732505c85067df3a7f7d7ebb5ce", + "size": 573, + "uri": "https://{canondata_backend}/1773845/77f8b008f626911d8af361315777d5aab3bab090/resource.tar.gz#test.test_pg_catalog-pg_shdescription-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_shdescription-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json index 6b9fa860c0c7..963232277195 100644 --- a/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json @@ -2191,9 +2191,9 @@ ], "test.test[pg_catalog-pg_auth_members-default.txt-Debug]": [ { - "checksum": "89087bb5c866532507e766caadeb1947", - "size": 462, - "uri": "https://{canondata_backend}/1784826/fe2bef548a55eb11e26daaded455ba74fda33a1b/resource.tar.gz#test.test_pg_catalog-pg_auth_members-default.txt-Debug_/opt.yql_patched" + "checksum": "67c1707f627eb1d62b8296d47ffeb054", + "size": 517, + "uri": "https://{canondata_backend}/1773845/3eb333049c811fbf89e9248a29fd2bbd21221d01/resource.tar.gz#test.test_pg_catalog-pg_auth_members-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_auth_members-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json index 07d26d440da0..1f53bc758ead 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json @@ -2129,9 +2129,9 @@ ], "test.test[pg_catalog-pg_auth_members-default.txt-Debug]": [ { - "checksum": "c64a1dbceb79a3d6167cc50da0addce8", - "size": 461, - "uri": "https://{canondata_backend}/1936947/581aa6d896ffe57e25bdb8006459e912860e61fa/resource.tar.gz#test.test_pg_catalog-pg_auth_members-default.txt-Debug_/opt.yql_patched" + "checksum": "adc22c478c42ba7e6a7251d2ed825f60", + "size": 516, + "uri": "https://{canondata_backend}/1773845/3f7cf07086245864aa149e34543610596d0da304/resource.tar.gz#test.test_pg_catalog-pg_auth_members-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_auth_members-default.txt-Plan]": [ @@ -2171,9 +2171,9 @@ ], "test.test[pg_catalog-pg_shdescription-default.txt-Debug]": [ { - "checksum": "a91c33b2710652de658a0b6a8a659550", - "size": 525, - "uri": "https://{canondata_backend}/1936947/581aa6d896ffe57e25bdb8006459e912860e61fa/resource.tar.gz#test.test_pg_catalog-pg_shdescription-default.txt-Debug_/opt.yql_patched" + "checksum": "8a53afd5bb0e83a0045775c9ebbf931a", + "size": 572, + "uri": "https://{canondata_backend}/1773845/3f7cf07086245864aa149e34543610596d0da304/resource.tar.gz#test.test_pg_catalog-pg_shdescription-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_shdescription-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json index b27ab4df740a..623b385398a6 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json @@ -2208,9 +2208,9 @@ ], "test.test[pg_catalog-pg_shdescription-default.txt-Debug]": [ { - "checksum": "f4707f2922a10bd5aabefea7774b1293", - "size": 468, - "uri": "https://{canondata_backend}/212715/e1bc1d6e31fa656365a738e65224a7c3f774bae6/resource.tar.gz#test.test_pg_catalog-pg_shdescription-default.txt-Debug_/opt.yql" + "checksum": "4d572f9434eef01810edf0263aa7f36b", + "size": 513, + "uri": "https://{canondata_backend}/1773845/5a0439864ed7e407e5caa5c6641c39e23943be50/resource.tar.gz#test.test_pg_catalog-pg_shdescription-default.txt-Debug_/opt.yql" } ], "test.test[pg_catalog-pg_shdescription-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json index c15ca4feb9f8..dc994993b394 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json @@ -1865,9 +1865,9 @@ ], "test.test[pg_catalog-pg_auth_members-default.txt-Debug]": [ { - "checksum": "b9e488d1c0e8e61af71e3a3e23ec1a82", - "size": 406, - "uri": "https://{canondata_backend}/1937367/c23590248e6f206ff1c4e65a8fd7d146beaddac2/resource.tar.gz#test.test_pg_catalog-pg_auth_members-default.txt-Debug_/opt.yql" + "checksum": "c668126253121973484a627cd64fa78b", + "size": 461, + "uri": "https://{canondata_backend}/1942415/8827dbc8399dcb7af3d3431163eb1b968f62bc07/resource.tar.gz#test.test_pg_catalog-pg_auth_members-default.txt-Debug_/opt.yql" } ], "test.test[pg_catalog-pg_auth_members-default.txt-Plan]": [ From b5d7351121df24ca77df1a013ce0449b57de0ab3 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Fri, 29 Mar 2024 20:32:18 +0000 Subject: [PATCH 3/3] fix --- ydb/library/yql/ast/yql_expr.cpp | 10 ++++++++++ ydb/library/yql/ast/yql_expr.h | 4 +--- ydb/library/yql/core/common_opt/yql_co_pgselect.cpp | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 827e73b9967a..0c403c31517f 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -3029,11 +3029,21 @@ bool TItemExprType::Validate(TPositionHandle position, TExprContext& ctx) const return Validate(ctx.GetPosition(position), ctx); } +TStringBuf TItemExprType::GetCleanName(bool isVirtual) const { + if (!isVirtual) { + return Name; + } + + YQL_ENSURE(Name.StartsWith(YqlVirtualPrefix)); + return Name.SubStr(YqlVirtualPrefix.size()); +} + const TItemExprType* TItemExprType::GetCleanItem(bool isVirtual, TExprContext& ctx) const { if (!isVirtual) { return this; } + YQL_ENSURE(Name.StartsWith(YqlVirtualPrefix)); return ctx.MakeType(Name.SubStr(YqlVirtualPrefix.size()), ItemType); } diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index 2ba6db361639..b81f121f32ed 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -465,9 +465,7 @@ class TItemExprType : public TTypeAnnotationNode { return Name; } - TStringBuf GetCleanName(bool isVirtual) const { - return isVirtual ? Name.SubStr(YqlVirtualPrefix.size()) : Name; - } + TStringBuf GetCleanName(bool isVirtual) const; const TTypeAnnotationNode* GetItemType() const { return ItemType; diff --git a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp index 422c9ebd0658..74e9754fa39d 100644 --- a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp @@ -1079,7 +1079,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE } else { auto type = read.GetTypeAnn()-> Cast()->GetItemType()->Cast(); - auto pos = type->FindItemI(column, nullptr); + auto pos = type->FindItem(column); foundColumn = pos.Defined(); } } @@ -1104,7 +1104,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE auto type = input->Tail().GetTypeAnn()-> Cast()->GetType()->Cast(); - auto pos = type->FindItemI(column, nullptr); + auto pos = type->FindItem(column); foundColumn = pos.Defined(); if (foundColumn) { x.second.first = inputIndex;