Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes clang-format usage #929

Merged
merged 4 commits into from
Jan 10, 2024
Merged

Fixes clang-format usage #929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6d1c7a8
test
Jan 10, 2024
8cd73c7
up formatting
Jan 10, 2024
360da12
Merge branch 'main' into restore-format
Jan 10, 2024
150c1d2
up ColumnLimit
Jan 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 36 additions & 24 deletions ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/core/kqp/ut/common/columnshard.h>
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/core/testlib/common_helper.h>
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>

namespace NKikimr {
namespace NKqp {
Expand All @@ -15,12 +15,11 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
constexpr size_t size_single_table = 13152;

const TVector<TTestHelper::TColumnSchema> schema = {
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8),
TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32)
};
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8),
TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32)};

class TOlapStatsController: public NYDBTest::NColumnShard::TController {
class TOlapStatsController : public NYDBTest::NColumnShard::TController {
public:
TDuration GetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override {
return TDuration::MilliSeconds(10);
Expand All @@ -32,7 +31,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {

Y_UNIT_TEST(AddRowsTableStandalone) {
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<TOlapStatsController>();

TKikimrSettings runnerSettings;
runnerSettings.WithSampleTables = false;

Expand All @@ -44,8 +43,8 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
testHelper.CreateTable(testTable);
{
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
for(size_t i=0; i<inserted_rows; i++) {

for (size_t i = 0; i < inserted_rows; i++) {
tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull();
}

Expand Down Expand Up @@ -78,21 +77,25 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
testTableStore.SetName("/Root/TableStoreTest").SetPrimaryKey({"id"}).SetSchema(schema);
testHelper.CreateTable(testTableStore);
TTestHelper::TColumnTable testTable;
testTable.SetName("/Root/TableStoreTest/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
testTable.SetName("/Root/TableStoreTest/ColumnTableTest")
.SetPrimaryKey({"id"})
.SetSharding({"id"})
.SetSchema(schema);
testHelper.CreateTable(testTable);

{
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
for(size_t i=0; i<inserted_rows; i++) {
for (size_t i = 0; i < inserted_rows; i++) {
tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull();
}
testHelper.InsertData(testTable, tableInserter);
}

Sleep(TDuration::Seconds(1));

auto settings = TDescribeTableSettings().WithTableStatistics(true);
auto describeResult = testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest", settings).GetValueSync();
auto describeResult =
testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest", settings).GetValueSync();

UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());

Expand All @@ -117,23 +120,32 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {

Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).SetPriority(NActors::NLog::PRI_DEBUG).Initialize();

for(size_t t=0; t<tables_in_store; t++) {
for (size_t t = 0; t < tables_in_store; t++) {
TTestHelper::TColumnTable testTable;
testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t)).SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t))
.SetPrimaryKey({"id"})
.SetSharding({"id"})
.SetSchema(schema);
testHelper.CreateTable(testTable);

TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
for(size_t i=0; i < t+ inserted_rows; i++) {
tableInserter.AddRow().Add(i + t * tables_in_store).Add("test_res_" + std::to_string(i + t * tables_in_store)).AddNull();
for (size_t i = 0; i < t + inserted_rows; i++) {
tableInserter.AddRow()
.Add(i + t * tables_in_store)
.Add("test_res_" + std::to_string(i + t * tables_in_store))
.AddNull();
}
testHelper.InsertData(testTable, tableInserter);;
testHelper.InsertData(testTable, tableInserter);
}

Sleep(TDuration::Seconds(20));

auto settings = TDescribeTableSettings().WithTableStatistics(true);
for(size_t t=0; t<tables_in_store; t++) {
auto describeResult = testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t), settings).GetValueSync();
for (size_t t = 0; t < tables_in_store; t++) {
auto describeResult =
testHelper.GetSession()
.DescribeTable("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t), settings)
.GetValueSync();
UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
const auto& description = describeResult.GetTableDescription();

Expand All @@ -142,5 +154,5 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
}
}

} // namespace NKqp
} // namespace NKikimr
} // namespace NKqp
} // namespace NKikimr
22 changes: 12 additions & 10 deletions ydb/core/tx/columnshard/columnshard.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,13 +293,14 @@ void TColumnShard::UpdateResourceMetrics(const TActorContext& ctx, const TUsage&
metrics->TryUpdate(ctx);
}

void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats * tabletStats) {
void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats,
::NKikimrTableStats::TTableStats* tabletStats) {
NOlap::TSnapshot lastIndexUpdate = TablesManager.GetPrimaryIndexSafe().LastUpdate();
auto activeIndexStats = indexStats.Active(); // data stats excluding inactive and evicted
auto activeIndexStats = indexStats.Active(); // data stats excluding inactive and evicted

if (activeIndexStats.Rows < 0 || activeIndexStats.Bytes < 0) {
LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows
<< " Bytes: " << activeIndexStats.Bytes << TabletID());
LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows << " Bytes: " << activeIndexStats.Bytes
<< TabletID());

activeIndexStats.Rows = (activeIndexStats.Rows < 0) ? 0 : activeIndexStats.Rows;
activeIndexStats.Bytes = (activeIndexStats.Bytes < 0) ? 0 : activeIndexStats.Bytes;
Expand All @@ -309,7 +310,7 @@ void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats, :
tabletStats->SetDataSize(activeIndexStats.Bytes + TabletCounters->Simple()[COUNTER_COMMITTED_BYTES].Get());

// TODO: we need row/dataSize counters for evicted data (managed by tablet but stored outside)
//tabletStats->SetIndexSize(); // TODO: calc size of internal tables
// tabletStats->SetIndexSize(); // TODO: calc size of internal tables

tabletStats->SetLastAccessTime(LastAccessTime.MilliSeconds());
tabletStats->SetLastUpdateTime(lastIndexUpdate.GetPlanStep());
Expand All @@ -330,7 +331,7 @@ void TColumnShard::FillTxTableStats(::NKikimrTableStats::TTableStats* tableStats
}

void TColumnShard::FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
ev->Record.SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
ev->Record.SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
ev->Record.SetGeneration(Executor()->Generation());
ev->Record.SetRound(StatsReportRound++);
ev->Record.SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId);
Expand All @@ -346,13 +347,14 @@ void TColumnShard::FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDa
}
}

void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
void TColumnShard::FillColumnTableStats(const TActorContext& ctx,
std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
if (!TablesManager.HasPrimaryIndex()) {
return;
}
const auto& tablesIndexStats = TablesManager.MutablePrimaryIndex().GetStats();
LOG_S_DEBUG("There are stats for " << tablesIndexStats.size() << " tables");
for(const auto& [tableLocalID, columnStats] : tablesIndexStats) {
for (const auto& [tableLocalID, columnStats] : tablesIndexStats) {
if (!columnStats) {
LOG_S_ERROR("SendPeriodicStats: empty stats");
continue;
Expand All @@ -362,7 +364,7 @@ void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_pt
periodicTableStats->SetDatashardId(TabletID());
periodicTableStats->SetTableLocalId(tableLocalID);

periodicTableStats->SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
periodicTableStats->SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
periodicTableStats->SetGeneration(Executor()->Generation());
periodicTableStats->SetRound(StatsReportRound++);
periodicTableStats->SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId);
Expand Down Expand Up @@ -411,4 +413,4 @@ void TColumnShard::SendPeriodicStats() {
NTabletPipe::SendData(ctx, StatsReportPipe, ev.release());
}

}
} // namespace NKikimr::NColumnShard
3 changes: 2 additions & 1 deletion ydb/core/tx/columnshard/columnshard_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -488,11 +488,12 @@ class TColumnShard
void SendPeriodicStats();
void FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev);
void FillColumnTableStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev);
void ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats * tabletStats);
void ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats* tabletStats);
void FillTxTableStats(::NKikimrTableStats::TTableStats* tableStats) const;

static TDuration GetControllerPeriodicWakeupActivationPeriod();
static TDuration GetControllerStatsReportInterval();

public:
const std::shared_ptr<NOlap::IStoragesManager>& GetStoragesManager() const {
return StoragesManager;
Expand Down
40 changes: 18 additions & 22 deletions ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class TTxStoreTableStats: public TTxStoreStats<TEvDataShard::TEvPeriodicTableSta
bool PersistSingleStats(const TPathId& pathId, const TStatsQueue<TEvDataShard::TEvPeriodicTableStats>::TItem& item, TTransactionContext& txc, const TActorContext& ctx) override;
void ScheduleNextBatch(const TActorContext& ctx) override;

template<typename T>
template <typename T>
TPartitionStats PrepareStats(const TActorContext& ctx, const T& rec) const;
};

Expand Down Expand Up @@ -124,10 +124,8 @@ THolder<TProposeRequest> MergeRequest(
return std::move(request);
}

template<typename T>
TPartitionStats TTxStoreTableStats::PrepareStats(
const TActorContext& ctx, const T& rec) const {

template <typename T>
TPartitionStats TTxStoreTableStats::PrepareStats(const TActorContext& ctx, const T& rec) const {
const auto& tableStats = rec.GetTableStats();
const auto& tabletMetrics = rec.GetTabletMetrics();

Expand Down Expand Up @@ -183,7 +181,8 @@ TPartitionStats TTxStoreTableStats::PrepareStats(
}

bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
const TStatsQueueItem<TEvDataShard::TEvPeriodicTableStats>& item, NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
const TStatsQueueItem<TEvDataShard::TEvPeriodicTableStats>& item,
NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
const auto& rec = item.Ev->Get()->Record;
const auto datashardId = TTabletId(rec.GetDatashardId());

Expand All @@ -208,19 +207,17 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
TShardIdx shardIdx = Self->TabletIdToShardIdx[datashardId];

LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"TTxStoreTableStats.PersistSingleStats: main stats from"
<< " datashardId(TabletID)=" << datashardId
<< " maps to shardIdx: " << shardIdx
<< ", pathId: " << pathId
<< ", pathId map=" << Self->PathsById[pathId]->Name
<< ", is column=" << isColumnTable
<< ", is olap=" << isOlapStore);
"TTxStoreTableStats.PersistSingleStats: main stats from"
<< " datashardId(TabletID)=" << datashardId << " maps to shardIdx: " << shardIdx
<< ", pathId: " << pathId << ", pathId map=" << Self->PathsById[pathId]->Name
<< ", is column=" << isColumnTable << ", is olap=" << isOlapStore);

const TPartitionStats newStats = PrepareStats(ctx, rec);

LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"Add stats from shard with datashardId(TabletID)=" << datashardId << ", pathId " << pathId.LocalPathId
<< ": RowCount " << newStats.RowCount << ", DataSize " << newStats.DataSize);
"Add stats from shard with datashardId(TabletID)=" << datashardId << ", pathId " << pathId.LocalPathId
<< ": RowCount " << newStats.RowCount << ", DataSize "
<< newStats.DataSize);

NIceDb::TNiceDb db(txc.DB);

Expand Down Expand Up @@ -261,29 +258,28 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
updateSubdomainInfo = true;

const auto tables = rec.GetTables();
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"OLAP store contains " << tables.size() << " tables.");
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "OLAP store contains " << tables.size() << " tables.");

for(const auto& table : tables) {
for (const auto& table : tables) {
const TPartitionStats newTableStats = PrepareStats(ctx, table);

const TPathId tablePathId = TPathId(TOwnerId(pathId.OwnerId), TLocalPathId(table.GetTableLocalId()));

if (Self->ColumnTables.contains(tablePathId)) {
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"add stats for exists table with pathId=" << tablePathId);
"add stats for exists table with pathId=" << tablePathId);

auto columnTable = Self->ColumnTables.TakeVerified(tablePathId);
columnTable->UpdateTableStats(tablePathId, newTableStats);
} else {
LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"failed add stats for table with pathId=" << tablePathId);
"failed add stats for table with pathId=" << tablePathId);
}
}

} else if (isColumnTable) {
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "PersistSingleStats: ColumnTable rec.GetColumnTables() size="
<< rec.GetTables().size());
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"PersistSingleStats: ColumnTable rec.GetColumnTables() size=" << rec.GetTables().size());

auto columnTable = Self->ColumnTables.TakeVerified(pathId);
oldAggrStats = columnTable->GetStats().Aggregated;
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ void TPathDescriber::DescribeOlapStore(TPathId pathId, TPathElement::TPtr pathEl
}

void TPathDescriber::DescribeColumnTable(TPathId pathId, TPathElement::TPtr pathEl) {
const auto tableInfo = Self->ColumnTables.GetVerified(pathId);
const auto tableInfo = Self->ColumnTables.GetVerified(pathId);
Y_UNUSED(pathEl);

auto* pathDescription = Result->Record.MutablePathDescription();
Expand Down
Loading