Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pg system columns, support for relations in pg_catalog #3325

Merged
merged 3 commits into from
Mar 29, 2024
Merged

pg system columns, support for relations in pg_catalog #3325

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6eb91d7
init
vitstn Mar 29, 2024
eac0e8c
tests
vitstn Mar 29, 2024
b5d7351
fix
vitstn Mar 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ydb/library/yql/ast/yql_expr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3029,6 +3029,14 @@ bool TItemExprType::Validate(TPositionHandle position, TExprContext& ctx) const
return Validate(ctx.GetPosition(position), ctx);
}

const TItemExprType* TItemExprType::GetCleanItem(bool isVirtual, TExprContext& ctx) const {
if (!isVirtual) {
return this;
}

return ctx.MakeType<TItemExprType>(Name.SubStr(YqlVirtualPrefix.size()), ItemType);
}

bool TMultiExprType::Validate(TPosition position, TExprContext& ctx) const {
if (Items.size() > Max<ui16>()) {
ctx.AddError(TIssue(position, TStringBuilder() << "Too many elements: " << Items.size()));
Expand Down
42 changes: 30 additions & 12 deletions ydb/library/yql/ast/yql_expr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ class TBlockExprType;
class TScalarExprType;

const size_t DefaultMistypeDistance = 3;
const TString YqlVirtualPrefix = "_yql_virtual_";

extern const TStringBuf ZeroString;

Expand Down Expand Up @@ -464,6 +465,10 @@ class TItemExprType : public TTypeAnnotationNode {
return Name;
}

TStringBuf GetCleanName(bool isVirtual) const {
return isVirtual ? Name.SubStr(YqlVirtualPrefix.size()) : Name;
}

const TTypeAnnotationNode* GetItemType() const {
return ItemType;
}
Expand All @@ -472,6 +477,8 @@ class TItemExprType : public TTypeAnnotationNode {
return GetName() == other.GetName() && GetItemType() == other.GetItemType();
}

const TItemExprType* GetCleanItem(bool isVirtual, TExprContext& ctx) const;

private:
const TStringBuf Name;
const TTypeAnnotationNode* ItemType;
Expand Down Expand Up @@ -532,24 +539,35 @@ class TStructExprType : public TTypeAnnotationNode {
return it - Items.begin();
}

TMaybe<ui32> FindItemI(const TStringBuf& name) const {
auto strict = FindItem(name);
if (strict) {
return strict;
}
TMaybe<ui32> FindItemI(const TStringBuf& name, bool* isVirtual) const {
for (ui32 v = 0; v < 2; ++v) {
if (isVirtual) {
*isVirtual = v > 0;
}

TMaybe<ui32> ret;
for (ui32 i = 0; i < Items.size(); ++i) {
if (AsciiEqualsIgnoreCase(name, Items[i]->GetName())) {
if (ret) {
return Nothing();
auto nameToSearch = (v ? YqlVirtualPrefix : "") + name;
auto strict = FindItem(nameToSearch);
if (strict) {
return strict;
}

TMaybe<ui32> ret;
for (ui32 i = 0; i < Items.size(); ++i) {
if (AsciiEqualsIgnoreCase(nameToSearch, Items[i]->GetName())) {
if (ret) {
return Nothing();
}

ret = i;
}
}

ret = i;
if (ret) {
return ret;
}
}

return ret;
return Nothing();
}

const TTypeAnnotationNode* FindItemType(const TStringBuf& name) const {
Expand Down
4 changes: 2 additions & 2 deletions ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1079,7 +1079,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE
} else {
auto type = read.GetTypeAnn()->
Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
auto pos = type->FindItem(column);
auto pos = type->FindItemI(column, nullptr);
foundColumn = pos.Defined();
}
}
Expand All @@ -1104,7 +1104,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE

auto type = input->Tail().GetTypeAnn()->
Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
auto pos = type->FindItem(column);
auto pos = type->FindItemI(column, nullptr);
foundColumn = pos.Defined();
if (foundColumn) {
x.second.first = inputIndex;
Expand Down
38 changes: 22 additions & 16 deletions ydb/library/yql/core/type_ann/type_ann_pg.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ IGraphTransformer::TStatus InferPgCommonType(TPositionHandle pos, const TExprNod
{
size_t j = 0;
for (const auto& col : *childColumnOrder) {
auto itemIdx = structType->FindItemI(col);
auto itemIdx = structType->FindItemI(col, nullptr);
YQL_ENSURE(itemIdx);

const auto* type = structType->GetItems()[*itemIdx]->GetItemType();
Expand Down Expand Up @@ -1725,7 +1725,8 @@ bool ScanColumns(TExprNode::TPtr root, TInputs& inputs, const THashSet<TString>&
}
}

auto pos = x.Type->FindItemI(node->Tail().Content());
bool isVirtual;
auto pos = x.Type->FindItemI(node->Tail().Content(), &isVirtual);
if (pos) {
foundAlias = x.Alias;
++matches;
Expand All @@ -1737,7 +1738,8 @@ bool ScanColumns(TExprNode::TPtr root, TInputs& inputs, const THashSet<TString>&
}

if (x.Priority == TInput::External) {
x.UsedExternalColumns.insert(TString(x.Type->GetItems()[*pos]->GetName()));
auto name = TString(x.Type->GetItems()[*pos]->GetCleanName(isVirtual));
x.UsedExternalColumns.insert(name);
}
}
}
Expand Down Expand Up @@ -1966,17 +1968,18 @@ void AddColumns(const TInputs& inputs, const bool* hasStar, const THashSet<TStri
continue;
}

auto pos = x.Type->FindItemI(ref);
bool isVirtual;
auto pos = x.Type->FindItemI(ref, &isVirtual);
if (pos) {
auto item = x.Type->GetItems()[*pos];
TString lcase = to_lower(TString(item->GetName()));
TString lcase = to_lower(TString(item->GetCleanName(isVirtual)));
if (auto it = usedInUsing.find(lcase); it != usedInUsing.end() && !present.contains(lcase)) {
items.push_back(ctx.MakeType<TItemExprType>(it->second, item->GetItemType()));
present.emplace(lcase);
}
item = AddAlias(x.Alias, item, ctx);
items.push_back(item);
usedRefs.insert(TString(item->GetName()));
usedRefs.insert(TString(item->GetCleanName(isVirtual)));
}
}

Expand All @@ -1986,7 +1989,7 @@ void AddColumns(const TInputs& inputs, const bool* hasStar, const THashSet<TStri
}

for (const auto& ref : qualifiedRefs->find(x.Alias)->second) {
auto pos = x.Type->FindItemI(ref);
auto pos = x.Type->FindItemI(ref, nullptr);
if (pos) {
auto item = x.Type->GetItems()[*pos];
item = AddAlias(x.Alias, item, ctx);
Expand Down Expand Up @@ -2110,7 +2113,7 @@ IGraphTransformer::TStatus RebuildLambdaColumns(const TExprNode::TPtr& root, con
}
}

auto pos = x.Type->FindItemI(node->Tail().Content());
auto pos = x.Type->FindItemI(node->Tail().Content(), nullptr);
if (pos) {
return ctx.Expr.Builder(node->Pos())
.Callable("Member")
Expand Down Expand Up @@ -3119,7 +3122,7 @@ bool GatherExtraSortColumns(const TExprNode& data, const TInputs& inputs, TExprN
continue;
}

auto pos = x.Type->FindItemI(column);
auto pos = x.Type->FindItemI(column, nullptr);
if (pos) {
index = inputIndex;
break;
Expand Down Expand Up @@ -3791,7 +3794,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
TVector<const TItemExprType*> newStructItems;
TColumnOrder newOrder;
for (ui32 i = 0; i < p->Child(2)->ChildrenSize(); ++i) {
auto pos = inputStructType->FindItemI((*columnOrder)[i]);
auto pos = inputStructType->FindItemI((*columnOrder)[i], nullptr);
YQL_ENSURE(pos);
auto type = inputStructType->GetItems()[*pos]->GetItemType();
newOrder.push_back(TString(p->Child(2)->Child(i)->Content()));
Expand Down Expand Up @@ -4138,11 +4141,12 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
} else {
int matchCount = 0;
for (size_t j = 0; j <= groupInputs.size() - 2; ++j) {
auto pos = groupInputs[j].Type->FindItemI(name);
bool isVirtual;
auto pos = groupInputs[j].Type->FindItemI(name, &isVirtual);
if (!pos) {
continue;
}
lrNames[0] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[j].Alias, groupInputs[j].Type->GetItems()[*pos]->GetName()));
lrNames[0] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[j].Alias, groupInputs[j].Type->GetItems()[*pos]->GetCleanName(isVirtual)));
++matchCount;
}
if (!matchCount) {
Expand All @@ -4154,13 +4158,14 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
return IGraphTransformer::TStatus::Error;
}
}
auto pos = rightInput.Type->FindItemI(name);
bool isVirtual;
auto pos = rightInput.Type->FindItemI(name, &isVirtual);
usedInUsingBefore.emplace(lcase);
if (!pos) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Can't find column: " << name));
return IGraphTransformer::TStatus::Error;
}
lrNames[1] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(rightInput.Alias, rightInput.Type->GetItems()[*pos]->GetName()));
lrNames[1] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(rightInput.Alias, rightInput.Type->GetItems()[*pos]->GetCleanName(isVirtual)));
nodes[colIdx] = ctx.Expr.NewList(inp->Pos(), std::move(lrNames));
}
TExprNode::TListType newJoin(4);
Expand Down Expand Up @@ -4616,9 +4621,10 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
const auto type = data.Child(j)->Tail().GetTypeAnn()->Cast<TTypeExprType>()->
GetType()->Cast<TStructExprType>();
for (const auto& col : x.UsedExternalColumns) {
auto pos = type->FindItemI(col);
bool isVirtual;
auto pos = type->FindItemI(col, &isVirtual);
YQL_ENSURE(pos);
items.push_back(type->GetItems()[*pos]);
items.push_back(type->GetItems()[*pos]->GetCleanItem(isVirtual, ctx.Expr));
}

auto effectiveType = ctx.Expr.MakeType<TStructExprType>(items);
Expand Down
21 changes: 17 additions & 4 deletions ydb/library/yql/parser/pg_catalog/catalog.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1604,17 +1604,18 @@ struct TCatalog {
"lo_close",
"lo_unlink"
}),
StaticTables({
AllStaticTables({
#include "pg_class.generated.h"
}),
AllStaticColumns({
#include "columns.generated.h"
})
{
THashSet<ui32> usedTableOids;
for (const auto& t : StaticTables) {
for (const auto& t : AllStaticTables) {
StaticColumns.insert(std::make_pair(t, TVector<TColumnInfo>()));
Y_ENSURE(usedTableOids.insert(t.Oid).first);
StaticTables.insert(std::make_pair(TTableInfoKey(t), t));
}

for (const auto& c: AllStaticColumns) {
Expand Down Expand Up @@ -1834,8 +1835,9 @@ struct TCatalog {
THashMap<TString, TVector<ui32>> AggregationsByName;
THashSet<TString> ProhibitedProcs;

TVector<TTableInfo> StaticTables;
TVector<TTableInfo> AllStaticTables;
TVector<TColumnInfo> AllStaticColumns;
THashMap<TTableInfoKey, TTableInfo> StaticTables;
THashMap<TTableInfoKey, TVector<TColumnInfo>> StaticColumns;
};

Expand Down Expand Up @@ -3079,12 +3081,23 @@ void EnumLanguages(std::function<void(ui32, const TLanguageDesc&)> f) {

const TVector<TTableInfo>& GetStaticTables() {
const auto& catalog = TCatalog::Instance();
return catalog.StaticTables;
return catalog.AllStaticTables;
}

const THashMap<TTableInfoKey, TVector<TColumnInfo>>& GetStaticColumns() {
const auto& catalog = TCatalog::Instance();
return catalog.StaticColumns;
}

const TTableInfo& LookupStaticTable(const TTableInfoKey& tableKey) {
const auto& catalog = TCatalog::Instance();
auto tablePtr = catalog.StaticTables.FindPtr(tableKey);
if (!tablePtr) {
throw yexception() << "No such table: " << tableKey.Schema << "." << tableKey.Name;
}

return *tablePtr;
}


}
2 changes: 1 addition & 1 deletion ydb/library/yql/parser/pg_catalog/catalog.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,8 +353,8 @@ struct TColumnInfo {
};

const TVector<TTableInfo>& GetStaticTables();
const TTableInfo& LookupStaticTable(const TTableInfoKey& tableKey);
const THashMap<TTableInfoKey, TVector<TColumnInfo>>& GetStaticColumns();

}

template <>
Expand Down
Loading
Loading