Skip to content

Commit

Permalink
fix usage of api
Browse files Browse the repository at this point in the history
  • Loading branch information
@swalrus1
swalrus1 committed Nov 19, 2024
1 parent 3536ce2 commit 15c255a
Show file tree
Hide file tree
Showing 13 changed files with 144 additions and 110 deletions.
25 changes: 1 addition & 24 deletions ydb/core/grpc_services/rpc_log_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,30 +508,7 @@ class TDescribeLogTableRPC : public TRpcSchemeRequestActor<TDescribeLogTableRPC,
}

if (tableDescription.HasTtlSettings() && tableDescription.GetTtlSettings().HasEnabled()) {
const auto& inTTL = tableDescription.GetTtlSettings().GetEnabled();

switch (inTTL.GetColumnUnit()) {
case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_date_type_column();
outTTL.set_column_name(inTTL.GetColumnName());
outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
break;
}

case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: {
auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_value_since_unix_epoch();
outTTL.set_column_name(inTTL.GetColumnName());
outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit()));
outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
break;
}

default:
break;
}
FillTtlSettings(*describeLogTableResult.mutable_ttl_settings(), tableDescription.GetTtlSettings().GetEnabled());
}

return ReplyWithResult(Ydb::StatusIds::SUCCESS, describeLogTableResult, ctx);
Expand Down
5 changes: 3 additions & 2 deletions ydb/core/kqp/provider/yql_kikimr_gateway.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,13 +283,14 @@ void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::T
if (!settings.ColumnUnit) {
auto& opts = *proto.mutable_date_type_column();
opts.set_column_name(settings.ColumnName);
opts.set_expire_after_seconds(settings.ExpireAfter.Seconds());
} else {
auto& opts = *proto.mutable_value_since_unix_epoch();
opts.set_column_name(settings.ColumnName);
opts.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(*settings.ColumnUnit));
opts.set_expire_after_seconds(settings.ExpireAfter.Seconds());
}
auto* deleteTier = proto.add_tiers();
deleteTier->set_evict_after_seconds(settings.ExpireAfter.Seconds());
deleteTier->mutable_delete_();
}

Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) {
Expand Down
4 changes: 3 additions & 1 deletion ydb/core/persqueue/writer/metadata_initializers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,10 @@ void TSrcIdMetaInitializer::DoPrepare(NInitializer::IInitializerInput::TPtr cont
auto* ttlSettings = request.mutable_ttl_settings();
auto* columnTtl = ttlSettings->mutable_value_since_unix_epoch();
columnTtl->set_column_name("AccessTime");
columnTtl->set_expire_after_seconds(1382400);
columnTtl->set_column_unit(Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS);
auto* deleteTier = ttlSettings->add_tiers();
deleteTier->mutable_delete_();
deleteTier->set_evict_after_seconds(1382400);
}

result.emplace_back(new NInitializer::TGenericTableModifier<NRequest::TDialogCreateTable>(request, "create"));
Expand Down
79 changes: 22 additions & 57 deletions ydb/core/ydb_convert/table_description.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,12 @@ static void FillStoragePool(TStoragePoolHolder* out, TAddStoragePoolFunc<TStorag
THashSet<EAlterOperationKind> GetAlterOperationKinds(const Ydb::Table::AlterTableRequest* req) {
THashSet<EAlterOperationKind> ops;

if (req->add_columns_size() || req->drop_columns_size() ||
req->alter_columns_size() ||
req->ttl_action_case() !=
Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET ||
req->has_alter_storage_settings() || req->add_column_families_size() ||
req->alter_column_families_size() || req->set_compaction_policy() ||
req->has_alter_partitioning_settings() ||
req->set_key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED ||
req->has_set_read_replicas_settings())
{
if (req->add_columns_size() || req->drop_columns_size() || req->alter_columns_size() ||
req->ttl_action_case() != Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET ||
req->tiering_action_case() != Ydb::Table::AlterTableRequest::TIERING_ACTION_NOT_SET || req->has_alter_storage_settings() ||
req->add_column_families_size() || req->alter_column_families_size() || req->set_compaction_policy() ||
req->has_alter_partitioning_settings() || req->set_key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED ||
req->has_set_read_replicas_settings()) {
ops.emplace(EAlterOperationKind::Common);
}

Expand Down Expand Up @@ -505,51 +501,6 @@ Ydb::Type* AddColumn<NKikimrSchemeOp::TColumnDescription>(Ydb::Table::ColumnMeta
return columnType;
}

template <typename TYdbProto, typename TTtl>
static void AddTtl(TYdbProto& out, const TTtl& inTTL) {
for (const auto& inTier : inTTL.GetTiers()) {
auto* outTier = out.mutable_ttl_settings()->add_tiers();
outTier->set_evict_after_seconds(inTier.GetEvictAfterSeconds());
switch (inTier.GetActionCase()) {
case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kDelete:
outTier->mutable_delete_();
break;
case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kEvictToExternalStorage:
outTier->mutable_evict_to_external_storage()->set_storage_name(inTier.GetEvictToExternalStorage().GetStorageName());
break;
case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::ACTION_NOT_SET:
break;
}
}

switch (inTTL.GetColumnUnit()) {
case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
auto& outTTL = *out.mutable_ttl_settings()->mutable_date_type_column();
outTTL.set_column_name(inTTL.GetColumnName());
break;
}

case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: {
auto& outTTL = *out.mutable_ttl_settings()->mutable_value_since_unix_epoch();
outTTL.set_column_name(inTTL.GetColumnName());
outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit()));
break;
}

default:
break;
}

if constexpr (std::is_same_v<TTtl, NKikimrSchemeOp::TTTLSettings::TEnabled>) {
if (inTTL.HasSysSettings() && inTTL.GetSysSettings().HasRunInterval()) {
out.mutable_ttl_settings()->set_run_interval_seconds(TDuration::FromValue(inTTL.GetSysSettings().GetRunInterval()).Seconds());
}
}
}

template <typename TYdbProto>
void FillColumnDescriptionImpl(TYdbProto& out,
NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) {
Expand Down Expand Up @@ -581,9 +532,13 @@ void FillColumnDescriptionImpl(TYdbProto& out,

if (in.HasTTLSettings()) {
if (in.GetTTLSettings().HasEnabled()) {
AddTtl(out, in.GetTTLSettings().GetEnabled());
FillTtlSettings(*out.mutable_ttl_settings(), in.GetTTLSettings().GetEnabled());
}
}

if (in.GetTTLSettings().HasUseTiering()) {
out.set_tiering(in.GetTTLSettings().GetUseTiering());
}
}

void FillColumnDescription(Ydb::Table::DescribeTableResult& out,
Expand Down Expand Up @@ -617,10 +572,14 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc

if (in.HasTtlSettings()) {
if (in.GetTtlSettings().HasEnabled()) {
AddTtl(out, in.GetTtlSettings().GetEnabled());
FillTtlSettings(*out.mutable_ttl_settings(), in.GetTtlSettings().GetEnabled());
}
}

if (in.GetTtlSettings().HasUseTiering()) {
out.set_tiering(in.GetTtlSettings().GetUseTiering());
}

out.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN);
}

Expand Down Expand Up @@ -832,6 +791,12 @@ bool BuildAlterColumnTableModifyScheme(const TString& path, const Ydb::Table::Al
} else if (req->has_drop_ttl_settings()) {
alterColumnTable->MutableAlterTtlSettings()->MutableDisabled();
}

if (req->has_set_tiering()) {
alterColumnTable->MutableAlterTtlSettings()->SetUseTiering(req->set_tiering());
} else if (req->has_drop_tiering()) {
alterColumnTable->MutableAlterTtlSettings()->SetUseTiering("");
}
}

return true;
Expand Down
63 changes: 53 additions & 10 deletions ydb/core/ydb_convert/table_settings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,10 +228,6 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc,
}
}

if (proto.tiering().size()) {
tableDesc.MutableTTLSettings()->SetUseTiering(proto.tiering());
}

if (proto.has_storage_settings()) {
TColumnFamilyManager families(tableDesc.MutablePartitionConfig());
if (!families.ApplyStorageSettings(proto.storage_settings(), &code, &error)) {
Expand Down Expand Up @@ -391,12 +387,6 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc,
tableDesc.MutableTTLSettings()->MutableDisabled();
}

if (proto.has_set_tiering()) {
tableDesc.MutableTTLSettings()->SetUseTiering(proto.set_tiering());
} else if (proto.has_drop_tiering()) {
tableDesc.MutableTTLSettings()->SetUseTiering("");
}

if (!changed && !hadPartitionConfig) {
tableDesc.ClearPartitionConfig();
}
Expand Down Expand Up @@ -460,4 +450,57 @@ bool FillIndexTablePartitioning(
return true;
}

template <typename TTtl>
void FillTtlSettingsImpl(Ydb::Table::TtlSettings& out, const TTtl& in) {
for (const auto& inTier : in.GetTiers()) {
auto* outTier = out.add_tiers();
outTier->set_evict_after_seconds(inTier.GetEvictAfterSeconds());
switch (inTier.GetActionCase()) {
case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kDelete:
outTier->mutable_delete_();
break;
case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kEvictToExternalStorage:
outTier->mutable_evict_to_external_storage()->set_storage_name(inTier.GetEvictToExternalStorage().GetStorageName());
break;
case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::ACTION_NOT_SET:
break;
}
}

switch (in.GetColumnUnit()) {
case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
auto& outTTL = *out.mutable_date_type_column();
outTTL.set_column_name(in.GetColumnName());
break;
}

case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS:
case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: {
auto& outTTL = *out.mutable_value_since_unix_epoch();
outTTL.set_column_name(in.GetColumnName());
outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(in.GetColumnUnit()));
break;
}

default:
break;
}

if constexpr (std::is_same_v<TTtl, NKikimrSchemeOp::TTTLSettings::TEnabled>) {
if (in.HasSysSettings() && in.GetSysSettings().HasRunInterval()) {
out.set_run_interval_seconds(TDuration::FromValue(in.GetSysSettings().GetRunInterval()).Seconds());
}
}
}

void FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TTTLSettings::TEnabled& in) {
FillTtlSettingsImpl(out, in);
}

void FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& in) {
FillTtlSettingsImpl(out, in);
}

} // namespace NKikimr
14 changes: 10 additions & 4 deletions ydb/core/ydb_convert/table_settings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out,
const Ydb::Table::AlterTableRequest& in,
Ydb::StatusIds::StatusCode& code, TString& error, bool changed);


// out
void FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TTTLSettings::TEnabled& in);
void FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& in);
// in
template <class TTtlSettingsEnabled>
bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in,
Ydb::StatusIds::StatusCode& code, TString& error)
Expand All @@ -38,7 +43,7 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
deleteTier->MutableDelete();
};

static const auto& fillColumnUnit = []<class TModeSettings> (TTtlSettingsEnabled& out, const TModeSettings& in) -> bool {
static const auto& fillColumnUnit = [&unsupported]<class TModeSettings> (TTtlSettingsEnabled& out, const TModeSettings& in) -> bool {
#define CASE_UNIT(type) \
case Ydb::Table::ValueSinceUnixEpochModeSettings::type: \
out.SetColumnUnit(NKikimrSchemeOp::TTTLSettings::type); \
Expand All @@ -51,8 +56,9 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
CASE_UNIT(UNIT_NANOSECONDS);
default:
return unsupported(TStringBuilder() << "Unsupported unit: "
<< static_cast<ui32>(in.value_since_unix_epoch().column_unit()));
<< static_cast<ui32>(in.column_unit()));
}
return true;

#undef CASE_UNIT
};
Expand Down Expand Up @@ -106,10 +112,10 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
std::optional<ui32> expireInSeconds = 0;
for (const auto& tier : out.GetTiers()) {
if (tier.HasDelete()) {
expireInSeconds = tier.GetEvictInSeconds();
expireInSeconds = tier.GetEvictAfterSeconds();
}
}
out.SetExpireInSeconds(expireInSeconds.value_or(std::numeric_limits<uint32_t>::max()));
out.SetExpireAfterSeconds(expireInSeconds.value_or(std::numeric_limits<uint32_t>::max()));

if constexpr (std::is_same_v<TTtlSettingsEnabled, NKikimrSchemeOp::TTTLSettings::TEnabled>) {
if (in.run_interval_seconds()) {
Expand Down
13 changes: 9 additions & 4 deletions ydb/public/api/protos/ydb_table.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,13 +485,13 @@ message ValueSinceUnixEpochModeSettings {
}

message DeprecatedDateTypeColumnModeSettings {
// Deprecated TTL mode for internal use only
// Deprecated TTL mode
string column_name = 1;
uint32 expire_after_seconds = 2;
}

message DeprecatedValueSinceUnixEpochModeSettings {
// Deprecated TTL mode for internal use only
// Deprecated TTL mode
string column_name = 1;
ValueSinceUnixEpochModeSettings.Unit column_unit = 2;
uint32 expire_after_seconds = 3;
Expand Down Expand Up @@ -654,7 +654,8 @@ message CreateTableRequest {
Ydb.FeatureFlag.Status key_bloom_filter = 16;
// Read replicas settings for table
ReadReplicasSettings read_replicas_settings = 17;
reserved 18; // tiering
// Tiering rules name. It specifies how data migrates from one tier (logical storage) to another.
string tiering = 18;
// Is temporary table
bool temporary = 19;
// Is table column or row oriented
Expand Down Expand Up @@ -733,7 +734,11 @@ message AlterTableRequest {
repeated string drop_changefeeds = 20;
// Rename existed index
repeated RenameIndexItem rename_indexes = 21;
reserved 22, 23; // tiering_action
// Setup or remove tiering
oneof tiering_action {
string set_tiering = 22;
google.protobuf.Empty drop_tiering = 23;
}
}

message AlterTableResponse {
Expand Down
2 changes: 2 additions & 0 deletions ydb/public/lib/experimental/ydb_logstore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,8 @@ void TLogTableDescription::SerializeTo(Ydb::LogStore::CreateLogTableRequest& req

if (TtlSettings) {
TtlSettings->SerializeTo(*request.mutable_ttl_settings());
} else if (TieringSettings) {
TieringSettings->SerializeTo(*request.mutable_tiering_settings());
}
}

Expand Down
20 changes: 20 additions & 0 deletions ydb/public/lib/experimental/ydb_logstore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,21 @@ struct TLogTableSharding {
TLogTableSharding(const Ydb::LogStore::DescribeLogTableResult& desc);
};

class TTieringSettings {
private:
TString TieringId;
public:
TTieringSettings(const TString& tieringId)
: TieringId(tieringId) {

}

void SerializeTo(Ydb::LogStore::TieringSettings& proto) const {
proto.set_tiering_id(TieringId);
}

};

class TLogTableDescription {
public:
TLogTableDescription(const TString& schemaPresetName, const TLogTableSharding& sharding);
Expand Down Expand Up @@ -185,11 +200,16 @@ class TLogTableDescription {
TtlSettings = settings;
return *this;
}
TLogTableDescription& SetTieringSettings(const TTieringSettings& settings) {
TieringSettings = settings;
return *this;
}
private:
const TString SchemaPresetName;
const TSchema Schema;
const TLogTableSharding Sharding;
TMaybe<TTtlSettings> TtlSettings;
TMaybe<TTieringSettings> TieringSettings;
TString Owner;
TVector<NScheme::TPermissions> Permissions;
TVector<NScheme::TPermissions> EffectivePermissions;
Expand Down
Loading

0 comments on commit 15c255a

Please sign in to comment.