From b948eec06ac61517750f9170ddf2d9aa89c9ade9 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Tue, 14 Feb 2023 11:35:02 +0800 Subject: [PATCH 01/24] tiflash support keyspace feature Signed-off-by: iosmanthus --- contrib/client-c | 2 +- contrib/kvproto | 2 +- dbms/src/Common/RedactHelpers.cpp | 2 +- dbms/src/Common/RedactHelpers.h | 4 +- dbms/src/Debug/MockRaftStoreProxy.cpp | 2 +- dbms/src/Debug/MockSchemaGetter.h | 2 + dbms/src/Debug/MockTiDB.cpp | 8 +-- dbms/src/Debug/dbgFuncSchema.cpp | 4 +- dbms/src/Debug/dbgFuncSchemaName.cpp | 2 +- dbms/src/Debug/dbgNaturalDag.cpp | 6 +- dbms/src/Debug/dbgQueryExecutor.cpp | 10 +-- dbms/src/Debug/dbgTools.cpp | 2 +- dbms/src/Flash/BatchCoprocessorHandler.cpp | 1 + dbms/src/Flash/Coprocessor/DAGContext.h | 14 ++++- .../Coprocessor/DAGStorageInterpreter.cpp | 11 ++-- dbms/src/Flash/CoprocessorHandler.cpp | 1 + dbms/src/Flash/Management/ManualCompact.cpp | 3 +- dbms/src/Interpreters/IDAsPathUpgrader.cpp | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 7 ++- dbms/src/Server/Server.cpp | 61 +++++++++++++------ dbms/src/Storages/DeltaMerge/RowKeyRange.cpp | 9 +-- dbms/src/Storages/DeltaMerge/RowKeyRange.h | 15 +++-- dbms/src/Storages/GCManager.cpp | 18 +++--- dbms/src/Storages/GCManager.h | 2 +- dbms/src/Storages/StorageDeltaMerge.cpp | 6 +- .../Storages/Transaction/ApplySnapshot.cpp | 12 ++-- dbms/src/Storages/Transaction/KVStore.cpp | 3 +- .../Storages/Transaction/KeyspaceSnapshot.cpp | 47 ++++++++++++++ .../Storages/Transaction/KeyspaceSnapshot.h | 49 +++++++++++++++ dbms/src/Storages/Transaction/LearnerRead.cpp | 2 +- .../Storages/Transaction/PartitionStreams.cpp | 15 +++-- .../Transaction/ProxyFFIStatusService.cpp | 26 ++++++-- dbms/src/Storages/Transaction/Region.cpp | 6 ++ dbms/src/Storages/Transaction/Region.h | 2 + .../Storages/Transaction/RegionRangeKeys.h | 2 + dbms/src/Storages/Transaction/RegionState.cpp | 13 +++- dbms/src/Storages/Transaction/RegionTable.cpp | 58 ++++++++++-------- dbms/src/Storages/Transaction/RegionTable.h | 14 ++--- dbms/src/Storages/Transaction/TMTStorages.cpp | 29 ++++++--- dbms/src/Storages/Transaction/TMTStorages.h | 12 ++-- dbms/src/Storages/Transaction/TiDB.cpp | 23 ++++++- dbms/src/Storages/Transaction/TiDB.h | 18 +++++- .../src/Storages/Transaction/TiKVKeyValue.cpp | 49 +++++++++++++++ dbms/src/Storages/Transaction/TiKVKeyValue.h | 7 +++ .../Storages/Transaction/TiKVRecordFormat.h | 14 ++++- dbms/src/Storages/Transaction/Types.h | 8 +++ dbms/src/TiDB/Schema/SchemaBuilder.cpp | 55 ++++++++++------- dbms/src/TiDB/Schema/SchemaBuilder.h | 8 ++- dbms/src/TiDB/Schema/SchemaGetter.cpp | 15 +++-- dbms/src/TiDB/Schema/SchemaGetter.h | 24 ++++---- dbms/src/TiDB/Schema/SchemaNameMapper.h | 50 +++++++++++++-- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 49 +++++++++------ dbms/src/TiDB/Schema/SchemaSyncService.h | 4 +- dbms/src/TiDB/Schema/SchemaSyncer.h | 6 +- dbms/src/TiDB/Schema/TiDBSchemaSyncer.h | 40 ++++++------ 55 files changed, 616 insertions(+), 240 deletions(-) create mode 100644 dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp create mode 100644 dbms/src/Storages/Transaction/KeyspaceSnapshot.h create mode 100644 dbms/src/Storages/Transaction/TiKVKeyValue.cpp diff --git a/contrib/client-c b/contrib/client-c index d6cc312f69d..b4ab3efa024 160000 --- a/contrib/client-c +++ b/contrib/client-c @@ -1 +1 @@ -Subproject commit d6cc312f69da3f09ac74cc9db3742be9cfd62595 +Subproject commit b4ab3efa024386ad028876907bdbab682cc2f2b3 diff --git a/contrib/kvproto b/contrib/kvproto index 1b2b4114103..579d6ab1bff 160000 --- a/contrib/kvproto +++ b/contrib/kvproto @@ -1 +1 @@ -Subproject commit 1b2b4114103afb06796b7e44f45f7e55133673c0 +Subproject commit 579d6ab1bfffd5e5ca71436dff59bcb5843188c4 diff --git a/dbms/src/Common/RedactHelpers.cpp b/dbms/src/Common/RedactHelpers.cpp index 39af6aa08f3..0d9a0d9a928 100644 --- a/dbms/src/Common/RedactHelpers.cpp +++ b/dbms/src/Common/RedactHelpers.cpp @@ -28,7 +28,7 @@ void Redact::setRedactLog(bool v) Redact::REDACT_LOG.store(v, std::memory_order_relaxed); } -std::string Redact::handleToDebugString(DB::HandleID handle) +std::string Redact::handleToDebugString(int64_t handle) { if (Redact::REDACT_LOG.load(std::memory_order_relaxed)) return "?"; diff --git a/dbms/src/Common/RedactHelpers.h b/dbms/src/Common/RedactHelpers.h index b1a681c80dc..d58f64b5f46 100644 --- a/dbms/src/Common/RedactHelpers.h +++ b/dbms/src/Common/RedactHelpers.h @@ -14,8 +14,6 @@ #pragma once -#include - #include #include @@ -29,7 +27,7 @@ class Redact public: static void setRedactLog(bool v); - static std::string handleToDebugString(DB::HandleID handle); + static std::string handleToDebugString(int64_t handle); static std::string keyToDebugString(const char * key, size_t size); static std::string keyToHexString(const char * key, size_t size); diff --git a/dbms/src/Debug/MockRaftStoreProxy.cpp b/dbms/src/Debug/MockRaftStoreProxy.cpp index 727b2a2d4c7..ad0cd486600 100644 --- a/dbms/src/Debug/MockRaftStoreProxy.cpp +++ b/dbms/src/Debug/MockRaftStoreProxy.cpp @@ -665,7 +665,7 @@ TableID MockRaftStoreProxy::bootstrap_table( UInt64 table_id = MockTiDB::instance().newTable("d", "t", columns, tso, "", "dt"); auto schema_syncer = tmt.getSchemaSyncer(); - schema_syncer->syncSchemas(ctx); + schema_syncer->syncSchemas(ctx, NullspaceID); this->table_id = table_id; return table_id; } diff --git a/dbms/src/Debug/MockSchemaGetter.h b/dbms/src/Debug/MockSchemaGetter.h index 677455e895a..63219e91107 100644 --- a/dbms/src/Debug/MockSchemaGetter.h +++ b/dbms/src/Debug/MockSchemaGetter.h @@ -71,6 +71,8 @@ struct MockSchemaGetter } return res; } + + KeyspaceID getKeyspaceID() const { return NullspaceID; } }; } // namespace DB diff --git a/dbms/src/Debug/MockTiDB.cpp b/dbms/src/Debug/MockTiDB.cpp index 61c359ec298..85cc349fa67 100644 --- a/dbms/src/Debug/MockTiDB.cpp +++ b/dbms/src/Debug/MockTiDB.cpp @@ -82,9 +82,9 @@ TablePtr MockTiDB::dropTableInternal(Context & context, const String & database_ tables_by_id.erase(partition.id); if (drop_regions) { - for (auto & e : region_table.getRegionsByTable(partition.id)) + for (auto & e : region_table.getRegionsByTable(NullspaceID, partition.id)) kvstore->mockRemoveRegion(e.first, region_table); - region_table.removeTable(partition.id); + region_table.removeTable(NullspaceID, partition.id); } } } @@ -94,9 +94,9 @@ TablePtr MockTiDB::dropTableInternal(Context & context, const String & database_ if (drop_regions) { - for (auto & e : region_table.getRegionsByTable(table->id())) + for (auto & e : region_table.getRegionsByTable(NullspaceID, table->id())) kvstore->mockRemoveRegion(e.first, region_table); - region_table.removeTable(table->id()); + region_table.removeTable(NullspaceID, table->id()); } return table; diff --git a/dbms/src/Debug/dbgFuncSchema.cpp b/dbms/src/Debug/dbgFuncSchema.cpp index 9ef07f16e8b..b6adeee48bd 100644 --- a/dbms/src/Debug/dbgFuncSchema.cpp +++ b/dbms/src/Debug/dbgFuncSchema.cpp @@ -66,7 +66,7 @@ void dbgFuncRefreshSchemas(Context & context, const ASTs &, DBGInvoker::Printer auto schema_syncer = tmt.getSchemaSyncer(); try { - schema_syncer->syncSchemas(context); + schema_syncer->syncSchemas(context, NullspaceID); } catch (Exception & e) { @@ -95,7 +95,7 @@ void dbgFuncGcSchemas(Context & context, const ASTs & args, DBGInvoker::Printer gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient()); else gc_safe_point = safeGet(typeid_cast(*args[0]).value); - service->gc(gc_safe_point); + service->gc(gc_safe_point, NullspaceID); output("schemas gc done"); } diff --git a/dbms/src/Debug/dbgFuncSchemaName.cpp b/dbms/src/Debug/dbgFuncSchemaName.cpp index a082b5438f3..6ed4c4213f2 100644 --- a/dbms/src/Debug/dbgFuncSchemaName.cpp +++ b/dbms/src/Debug/dbgFuncSchemaName.cpp @@ -173,7 +173,7 @@ void dbgFuncGetPartitionTablesTiflashReplicaCount(Context & context, const ASTs for (const auto & part_def : table_info.partition.definitions) { auto paritition_table_info = table_info.producePartitionTableInfo(part_def.id, name_mapper); - auto partition_storage = context.getTMTContext().getStorages().get(paritition_table_info->id); + auto partition_storage = context.getTMTContext().getStorages().get(NullspaceID, paritition_table_info->id); fmt_buf.append((std::to_string(partition_storage->getTableInfo().replica_info.count))); fmt_buf.append("/"); } diff --git a/dbms/src/Debug/dbgNaturalDag.cpp b/dbms/src/Debug/dbgNaturalDag.cpp index 46adf58bba9..8af9acb6446 100644 --- a/dbms/src/Debug/dbgNaturalDag.cpp +++ b/dbms/src/Debug/dbgNaturalDag.cpp @@ -146,7 +146,7 @@ void NaturalDag::loadTables(const NaturalDag::JSONObjectPtr & obj) table.id = id; auto tbl_json = td_json->getObject(std::to_string(id)); auto meta_json = tbl_json->getObject(TABLE_META); - table.meta = TiDB::TableInfo(meta_json); + table.meta = TiDB::TableInfo(meta_json, NullspaceID); auto regions_json = tbl_json->getArray(TABLE_REGIONS); for (const auto & region_json : *regions_json) { @@ -208,7 +208,7 @@ void NaturalDag::buildTables(Context & context) auto & table = it.second; auto meta = table.meta; MockTiDB::instance().addTable(db_name, std::move(meta)); - schema_syncer->syncSchemas(context); + schema_syncer->syncSchemas(context, NullspaceID); for (auto & region : table.regions) { metapb::Region region_pb; @@ -243,7 +243,7 @@ void NaturalDag::buildDatabase(Context & context, SchemaSyncerPtr & schema_synce MockTiDB::instance().dropDB(context, db_name, true); } MockTiDB::instance().newDataBase(db_name); - schema_syncer->syncSchemas(context); + schema_syncer->syncSchemas(context, NullspaceID); } void NaturalDag::build(Context & context) diff --git a/dbms/src/Debug/dbgQueryExecutor.cpp b/dbms/src/Debug/dbgQueryExecutor.cpp index 359aa833f25..7027cf99588 100644 --- a/dbms/src/Debug/dbgQueryExecutor.cpp +++ b/dbms/src/Debug/dbgQueryExecutor.cpp @@ -180,7 +180,7 @@ BlockInputStreamPtr executeMPPQuery(Context & context, const DAGProperties & pro for (const auto & partition : table_info->partition.definitions) { const auto partition_id = partition.id; - auto regions = context.getTMTContext().getRegionTable().getRegionsByTable(partition_id); + auto regions = context.getTMTContext().getRegionTable().getRegionsByTable(NullspaceID, partition_id); for (size_t i = 0; i < regions.size(); ++i) { if ((current_region_size + i) % properties.mpp_partition_num != static_cast(task.partition_id)) @@ -201,7 +201,7 @@ BlockInputStreamPtr executeMPPQuery(Context & context, const DAGProperties & pro } else { - auto regions = context.getTMTContext().getRegionTable().getRegionsByTable(table_id); + auto regions = context.getTMTContext().getRegionTable().getRegionsByTable(NullspaceID, table_id); if (regions.size() < static_cast(properties.mpp_partition_num)) throw Exception("Not supported: table region num less than mpp partition num"); for (size_t i = 0; i < regions.size(); ++i) @@ -228,7 +228,7 @@ BlockInputStreamPtr executeNonMPPQuery(Context & context, RegionID region_id, co RegionPtr region; if (region_id == InvalidRegionID) { - auto regions = context.getTMTContext().getRegionTable().getRegionsByTable(table_id); + auto regions = context.getTMTContext().getRegionTable().getRegionsByTable(NullspaceID, table_id); if (regions.empty()) throw Exception("No region for table", ErrorCodes::BAD_ARGUMENTS); region = regions[0].second; @@ -306,7 +306,7 @@ tipb::SelectResponse executeDAGRequest(Context & context, const tipb::DAGRequest table_regions_info.local_regions.emplace(region_id, RegionInfo(region_id, region_version, region_conf_version, std::move(key_ranges), nullptr)); - DAGContext dag_context(dag_request, std::move(tables_regions_info), "", false, log); + DAGContext dag_context(dag_request, std::move(tables_regions_info), NullspaceID, "", false, log); context.setDAGContext(&dag_context); DAGDriver driver(context, start_ts, DEFAULT_UNSPECIFIED_SCHEMA_VERSION, &dag_response, true); @@ -334,7 +334,7 @@ bool runAndCompareDagReq(const coprocessor::Request & req, const coprocessor::Re auto & table_regions_info = tables_regions_info.getSingleTableRegions(); table_regions_info.local_regions.emplace(region_id, RegionInfo(region_id, region->version(), region->confVer(), std::move(key_ranges), nullptr)); - DAGContext dag_context(dag_request, std::move(tables_regions_info), "", false, log); + DAGContext dag_context(dag_request, std::move(tables_regions_info), NullspaceID, "", false, log); context.setDAGContext(&dag_context); DAGDriver driver(context, properties.start_ts, DEFAULT_UNSPECIFIED_SCHEMA_VERSION, &dag_response, true); driver.execute(); diff --git a/dbms/src/Debug/dbgTools.cpp b/dbms/src/Debug/dbgTools.cpp index 854d8a18bd5..04336b71a24 100644 --- a/dbms/src/Debug/dbgTools.cpp +++ b/dbms/src/Debug/dbgTools.cpp @@ -549,7 +549,7 @@ Int64 concurrentRangeOperate( { TMTContext & tmt = context.getTMTContext(); - for (auto && [_, r] : tmt.getRegionTable().getRegionsByTable(table_info.id)) + for (auto && [_, r] : tmt.getRegionTable().getRegionsByTable(NullspaceID, table_info.id)) { std::ignore = _; if (r == nullptr) diff --git a/dbms/src/Flash/BatchCoprocessorHandler.cpp b/dbms/src/Flash/BatchCoprocessorHandler.cpp index 5633d91a463..f3db61c4d4c 100644 --- a/dbms/src/Flash/BatchCoprocessorHandler.cpp +++ b/dbms/src/Flash/BatchCoprocessorHandler.cpp @@ -72,6 +72,7 @@ grpc::Status BatchCoprocessorHandler::execute() DAGContext dag_context( dag_request, std::move(tables_regions_info), + cop_request->context().keyspace_id(), cop_context.db_context.getClientInfo().current_address.toString(), /*is_batch_cop=*/true, Logger::get("BatchCoprocessorHandler")); diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 37562eeee96..4da71ed7622 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -127,7 +127,12 @@ class DAGContext { public: // for non-mpp(cop/batchCop) - explicit DAGContext(const tipb::DAGRequest & dag_request_, TablesRegionsInfo && tables_regions_info_, const String & tidb_host_, bool is_batch_cop_, LoggerPtr log_) + explicit DAGContext(const tipb::DAGRequest & dag_request_, + TablesRegionsInfo && tables_regions_info_, + KeyspaceID keyspace_id_, + const String & tidb_host_, + bool is_batch_cop_, + LoggerPtr log_) : dag_request(&dag_request_) , dummy_query_string(dag_request->DebugString()) , dummy_ast(makeDummyQuery()) @@ -143,6 +148,7 @@ class DAGContext , max_recorded_error_count(getMaxErrorCount(*dag_request)) , warnings(max_recorded_error_count) , warning_count(0) + , keyspace_id(keyspace_id_) { assert(dag_request->has_root_executor() || dag_request->executors_size() > 0); return_executor_id = dag_request->root_executor().has_executor_id() || dag_request->executors(0).has_executor_id(); @@ -166,6 +172,7 @@ class DAGContext , max_recorded_error_count(getMaxErrorCount(*dag_request)) , warnings(max_recorded_error_count) , warning_count(0) + , keyspace_id(meta_.keyspace_id()) { assert(dag_request->has_root_executor() && dag_request->root_executor().has_executor_id()); // only mpp task has join executor. @@ -335,6 +342,8 @@ class DAGContext void addTableLock(const TableLockHolder & lock) { table_locks.push_back(lock); } + KeyspaceID getKeyspaceID() const { return keyspace_id; } + const tipb::DAGRequest * dag_request; /// Some existing code inherited from Clickhouse assume that each query must have a valid query string and query ast, /// dummy_query_string and dummy_ast is used for that @@ -416,6 +425,9 @@ class DAGContext // In disaggregated tiflash mode, table_scan in tiflash_compute node will be converted ExchangeReceiver. // Record here so we can add to receiver_set and cancel/close it. std::optional> disaggregated_compute_exchange_receiver; + + // The keyspace that the DAG request from + const KeyspaceID keyspace_id = NullspaceID; }; } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 6c2cf2a2860..fb8a6040fd0 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -765,10 +765,11 @@ void DAGStorageInterpreter::buildLocalStreams(DAGPipeline & pipeline, size_t max std::unordered_map DAGStorageInterpreter::getAndLockStorages(Int64 query_schema_version) { + auto keyspace_id = context.getDAGContext()->getKeyspaceID(); std::unordered_map storages_with_lock; if (unlikely(query_schema_version == DEFAULT_UNSPECIFIED_SCHEMA_VERSION)) { - auto logical_table_storage = tmt.getStorages().get(logical_table_id); + auto logical_table_storage = tmt.getStorages().get(keyspace_id, logical_table_id); if (!logical_table_storage) { throw TiFlashException(fmt::format("Table {} doesn't exist.", logical_table_id), Errors::Table::NotExists); @@ -778,7 +779,7 @@ std::unordered_map DAG { for (auto const physical_table_id : table_scan.getPhysicalTableIDs()) { - auto physical_table_storage = tmt.getStorages().get(physical_table_id); + auto physical_table_storage = tmt.getStorages().get(keyspace_id,physical_table_id); if (!physical_table_storage) { throw TiFlashException(fmt::format("Table {} doesn't exist.", physical_table_id), Errors::Table::NotExists); @@ -789,14 +790,14 @@ std::unordered_map DAG return storages_with_lock; } - auto global_schema_version = tmt.getSchemaSyncer()->getCurrentVersion(); + auto global_schema_version = tmt.getSchemaSyncer()->getCurrentVersion(keyspace_id); /// Align schema version under the read lock. /// Return: [storage, table_structure_lock, storage_schema_version, ok] auto get_and_lock_storage = [&](bool schema_synced, TableID table_id) -> std::tuple { /// Get storage in case it's dropped then re-created. // If schema synced, call getTable without try, leading to exception on table not existing. - auto table_store = tmt.getStorages().get(table_id); + auto table_store = tmt.getStorages().get(keyspace_id,table_id); if (!table_store) { if (schema_synced) @@ -891,7 +892,7 @@ std::unordered_map DAG auto sync_schema = [&] { auto start_time = Clock::now(); GET_METRIC(tiflash_schema_trigger_count, type_cop_read).Increment(); - tmt.getSchemaSyncer()->syncSchemas(context); + tmt.getSchemaSyncer()->syncSchemas(context, dagContext().getKeyspaceID()); auto schema_sync_cost = std::chrono::duration_cast(Clock::now() - start_time).count(); LOG_INFO(log, "Table {} schema sync cost {}ms.", logical_table_id, schema_sync_cost); }; diff --git a/dbms/src/Flash/CoprocessorHandler.cpp b/dbms/src/Flash/CoprocessorHandler.cpp index f5f4b313363..071be72ca14 100644 --- a/dbms/src/Flash/CoprocessorHandler.cpp +++ b/dbms/src/Flash/CoprocessorHandler.cpp @@ -109,6 +109,7 @@ grpc::Status CoprocessorHandler::execute() DAGContext dag_context( dag_request, std::move(tables_regions_info), + cop_request->context().keyspace_id(), cop_context.db_context.getClientInfo().current_address.toString(), /*is_batch_cop=*/false, Logger::get("CoprocessorHandler")); diff --git a/dbms/src/Flash/Management/ManualCompact.cpp b/dbms/src/Flash/Management/ManualCompact.cpp index c79bebdcc2c..95ba868e6b6 100644 --- a/dbms/src/Flash/Management/ManualCompact.cpp +++ b/dbms/src/Flash/Management/ManualCompact.cpp @@ -100,7 +100,8 @@ grpc::Status ManualCompactManager::doWorkWithCatch(const ::kvrpcpb::CompactReque grpc::Status ManualCompactManager::doWork(const ::kvrpcpb::CompactRequest * request, ::kvrpcpb::CompactResponse * response) { const auto & tmt_context = global_context.getTMTContext(); - auto storage = tmt_context.getStorages().get(request->physical_table_id()); + // TODO(iosmanthus): support compact keyspace tables; + auto storage = tmt_context.getStorages().get(NullspaceID, request->physical_table_id()); if (storage == nullptr) { response->mutable_error()->mutable_err_physical_table_not_exist(); diff --git a/dbms/src/Interpreters/IDAsPathUpgrader.cpp b/dbms/src/Interpreters/IDAsPathUpgrader.cpp index e253dfdb66d..3184a1a43a8 100644 --- a/dbms/src/Interpreters/IDAsPathUpgrader.cpp +++ b/dbms/src/Interpreters/IDAsPathUpgrader.cpp @@ -503,7 +503,7 @@ std::vector IDAsPathUpgrader::fetchInfosFromTiDB() const try { auto schema_syncer = global_context.getTMTContext().getSchemaSyncer(); - return schema_syncer->fetchAllDBs(); + return schema_syncer->fetchAllDBs(NullspaceID); } catch (Poco::Exception & e) { diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 35d71695914..a04ca786a5d 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -227,7 +227,7 @@ void InterpreterSelectQuery::getAndLockStorageWithSchemaVersion(const String & d const String qualified_name = database_name + "." + table_name; /// Get current schema version in schema syncer for a chance to shortcut. - const auto global_schema_version = context.getTMTContext().getSchemaSyncer()->getCurrentVersion(); + const auto global_schema_version = context.getTMTContext().getSchemaSyncer()->getCurrentVersion(NullspaceID); /// Lambda for get storage, then align schema version under the read lock. auto get_and_lock_storage = [&](bool schema_synced) -> std::tuple { @@ -298,7 +298,10 @@ void InterpreterSelectQuery::getAndLockStorageWithSchemaVersion(const String & d { log_schema_version("not OK, syncing schemas."); auto start_time = Clock::now(); - context.getTMTContext().getSchemaSyncer()->syncSchemas(context); + // Since InterpreterSelectQuery will only be trigger while using ClickHouse client, + // and we do not support keyspace feature for ClickHouse interface, + // we could use nullspace id here safely. + context.getTMTContext().getSchemaSyncer()->syncSchemas(context, NullspaceID); auto schema_sync_cost = std::chrono::duration_cast(Clock::now() - start_time).count(); LOG_DEBUG(log, "Table {} schema sync cost {}ms.", qualified_name, schema_sync_cost); diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 4dac558477b..76493bf5224 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -295,7 +295,7 @@ struct TiFlashProxyConfig const std::string TiFlashProxyConfig::config_prefix = "flash.proxy"; -pingcap::ClusterConfig getClusterConfig(TiFlashSecurityConfigPtr security_config, const TiFlashRaftConfig & raft_config, const LoggerPtr & log) +pingcap::ClusterConfig getClusterConfig(TiFlashSecurityConfigPtr security_config, const TiFlashRaftConfig & raft_config, const int api_version, const LoggerPtr & log) { pingcap::ClusterConfig config; config.tiflash_engine_key = raft_config.engine_key; @@ -304,7 +304,18 @@ pingcap::ClusterConfig getClusterConfig(TiFlashSecurityConfigPtr security_config config.ca_path = ca_path; config.cert_path = cert_path; config.key_path = key_path; - LOG_INFO(log, "update cluster config, ca_path: {}, cert_path: {}, key_path: {}", ca_path, cert_path, key_path); + switch (api_version) + { + case 1: + config.api_version = kvrpcpb::APIVersion::V1; + break; + case 2: + config.api_version = kvrpcpb::APIVersion::V2; + break; + default: + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Invalid api version {}", api_version); + } + LOG_INFO(log, "update cluster config, ca_path: {}, cert_path: {}, key_path: {}, api_version: {}", ca_path, cert_path, key_path, config.api_version); return config; } @@ -843,6 +854,8 @@ int Server::main(const std::vector & /*args*/) TiFlashProxyConfig proxy_conf(config()); + auto api_version = config().getInt("api_version", 1); + EngineStoreServerWrap tiflash_instance_wrap{}; auto helper = GetEngineStoreServerHelper( &tiflash_instance_wrap); @@ -1131,7 +1144,7 @@ int Server::main(const std::vector & /*args*/) if (updated) { auto raft_config = TiFlashRaftConfig::parseSettings(*config, log); - auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, log); + auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, api_version, log); global_context->getTMTContext().updateSecurityConfig(std::move(raft_config), std::move(cluster_config)); LOG_DEBUG(log, "TMTContext updated security config"); } @@ -1188,7 +1201,7 @@ int Server::main(const std::vector & /*args*/) { /// create TMTContext - auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, log); + auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, api_version, log); global_context->createTMTContext(raft_config, std::move(cluster_config)); global_context->getTMTContext().reloadConfig(config()); } @@ -1226,26 +1239,30 @@ int Server::main(const std::vector & /*args*/) if (!global_context->isDisaggregatedComputeMode()) { /// Then, sync schemas with TiDB, and initialize schema sync service. - for (int i = 0; i < 60; i++) // retry for 3 mins + /// If in API V2 mode, each keyspace's schema is fetch lazily. + if (api_version == 1) { - try + for (int i = 0; i < 60; i++) // retry for 3 mins { - global_context->getTMTContext().getSchemaSyncer()->syncSchemas(*global_context); - break; - } - catch (Poco::Exception & e) - { - const int wait_seconds = 3; - LOG_ERROR( - log, - "Bootstrap failed because sync schema error: {}\nWe will sleep for {}" - " seconds and try again.", - e.displayText(), - wait_seconds); - ::sleep(wait_seconds); + try + { + global_context->getTMTContext().getSchemaSyncer()->syncSchemas(*global_context, NullspaceID); + break; + } + catch (Poco::Exception & e) + { + const int wait_seconds = 3; + LOG_ERROR( + log, + "Bootstrap failed because sync schema error: {}\nWe will sleep for {}" + " seconds and try again.", + e.displayText(), + wait_seconds); + ::sleep(wait_seconds); + } } + LOG_DEBUG(log, "Sync schemas done."); } - LOG_DEBUG(log, "Sync schemas done."); initStores(*global_context, log, storage_config.lazily_init_store); @@ -1363,6 +1380,10 @@ int Server::main(const std::vector & /*args*/) auto & tmt_context = global_context->getTMTContext(); if (proxy_conf.is_proxy_runnable) { + while(!tmt_context.getKVCluster()->pd_client->isClusterBootstrapped()) { + LOG_INFO(log, "Waiting for PD cluster to be bootstrapped"); + sleep(1); + } tiflash_instance_wrap.tmt = &tmt_context; LOG_INFO(log, "Let tiflash proxy start all services"); tiflash_instance_wrap.status = EngineStoreServerStatus::Running; diff --git a/dbms/src/Storages/DeltaMerge/RowKeyRange.cpp b/dbms/src/Storages/DeltaMerge/RowKeyRange.cpp index 9e2ea7ffbad..d7de4b701c8 100644 --- a/dbms/src/Storages/DeltaMerge/RowKeyRange.cpp +++ b/dbms/src/Storages/DeltaMerge/RowKeyRange.cpp @@ -55,18 +55,19 @@ RowKeyValue RowKeyValueRef::toRowKeyValue() const } } -std::unordered_map RowKeyRange::table_min_max_data; +std::unordered_map> RowKeyRange::table_min_max_data; std::shared_mutex RowKeyRange::table_mutex; -const RowKeyRange::TableRangeMinMax & RowKeyRange::getTableMinMaxData(TableID table_id, bool is_common_handle) +const RowKeyRange::TableRangeMinMax & RowKeyRange::getTableMinMaxData(KeyspaceID keyspace_id, TableID table_id, bool is_common_handle) { + auto keyspace_table_id = KeyspaceTableID{keyspace_id, table_id}; { std::shared_lock lock(table_mutex); - if (auto it = table_min_max_data.find(table_id); it != table_min_max_data.end()) + if (auto it = table_min_max_data.find(keyspace_table_id); it != table_min_max_data.end()) return it->second; } std::unique_lock lock(table_mutex); - return table_min_max_data.try_emplace(table_id, table_id, is_common_handle).first->second; + return table_min_max_data.try_emplace(keyspace_table_id, keyspace_id, table_id, is_common_handle).first->second; } template diff --git a/dbms/src/Storages/DeltaMerge/RowKeyRange.h b/dbms/src/Storages/DeltaMerge/RowKeyRange.h index acbcda2ec73..5e5cf428faa 100644 --- a/dbms/src/Storages/DeltaMerge/RowKeyRange.h +++ b/dbms/src/Storages/DeltaMerge/RowKeyRange.h @@ -414,9 +414,11 @@ struct RowKeyRange HandleValuePtr min; HandleValuePtr max; - TableRangeMinMax(TableID table_id, bool is_common_handle) + TableRangeMinMax(KeyspaceID keyspace_id, TableID table_id, bool is_common_handle) { WriteBufferFromOwnString ss; + auto ks_pfx = DecodedTiKVKey::makeKeyspacePrefix(keyspace_id); + ss.write(ks_pfx.data(), ks_pfx.size()); ss.write('t'); EncodeInt64(table_id, ss); ss.write('_'); @@ -436,9 +438,9 @@ struct RowKeyRange }; /// maybe use a LRU cache in case there are massive tables - static std::unordered_map table_min_max_data; + static std::unordered_map> table_min_max_data; static std::shared_mutex table_mutex; - static const TableRangeMinMax & getTableMinMaxData(TableID table_id, bool is_common_handle); + static const TableRangeMinMax & getTableMinMaxData(KeyspaceID keyspace_id, TableID table_id, bool is_common_handle); RowKeyRange(const RowKeyValue & start_, const RowKeyValue & end_, bool is_common_handle_, size_t rowkey_column_size_) : is_common_handle(is_common_handle_) @@ -751,7 +753,8 @@ struct RowKeyRange { auto & start_key = *raw_keys.first; auto & end_key = *raw_keys.second; - const auto & table_range_min_max = getTableMinMaxData(table_id, is_common_handle); + auto keyspace_id = start_key.getKeyspaceID(); + const auto & table_range_min_max = getTableMinMaxData(keyspace_id, table_id, is_common_handle); RowKeyValue start_value, end_value; if (start_key.compare(*table_range_min_max.min) <= 0) { @@ -763,7 +766,7 @@ struct RowKeyRange else { start_value = RowKeyValue(is_common_handle, - std::make_shared(start_key.begin() + RecordKVFormat::RAW_KEY_NO_HANDLE_SIZE, start_key.end())); + std::make_shared(RecordKVFormat::getRawTiDBPKView(start_key))); } if (end_key.compare(*table_range_min_max.max) >= 0) { @@ -774,7 +777,7 @@ struct RowKeyRange } else end_value = RowKeyValue(is_common_handle, - std::make_shared(end_key.begin() + RecordKVFormat::RAW_KEY_NO_HANDLE_SIZE, end_key.end())); + std::make_shared(RecordKVFormat::getRawTiDBPKView(end_key))); return RowKeyRange(start_value, end_value, is_common_handle, rowkey_column_size); } else diff --git a/dbms/src/Storages/GCManager.cpp b/dbms/src/Storages/GCManager.cpp index 83725c2de2e..eea5db3d22a 100644 --- a/dbms/src/Storages/GCManager.cpp +++ b/dbms/src/Storages/GCManager.cpp @@ -43,15 +43,15 @@ bool GCManager::work() return false; } - LOG_DEBUG(log, "Start GC with table id: {}", next_table_id); + LOG_DEBUG(log, "Start GC with keyspace id: {}, table id: {}", next_keyspace_table_id.first, next_keyspace_table_id.second); // Get a storage snapshot with weak_ptrs first // TODO: avoid gc on storage which have no data? - std::map> storages; - for (const auto & [table_id, storage] : global_context.getTMTContext().getStorages().getAllStorage()) - storages.emplace(table_id, storage); + std::map> storages; + for (const auto & [ks_tb_id, storage] : global_context.getTMTContext().getStorages().getAllStorage()) + storages.emplace(ks_tb_id, storage); auto iter = storages.begin(); - if (next_table_id != InvalidTableID) - iter = storages.lower_bound(next_table_id); + if (next_keyspace_table_id != KeyspaceTableID{NullspaceID, InvalidTableID}) + iter = storages.lower_bound(next_keyspace_table_id); UInt64 checked_storage_num = 0; while (true) @@ -79,7 +79,7 @@ bool GCManager::work() // do not acquire structure lock on the storage. auto gc_segments_num = storage->onSyncGc(gc_segments_limit, DM::GCOptions::newAll()); gc_segments_limit = gc_segments_limit - gc_segments_num; - LOG_TRACE(log, "GCManager gc {} segments of table {}", gc_segments_num, storage->getTableInfo().id); + LOG_TRACE(log, "GCManager gc {} segments of keyspace {}, table {}", gc_segments_num, storage->getTableInfo().keyspace_id, storage->getTableInfo().id); // Reach the limit on the number of segments to be gc, stop here if (gc_segments_limit <= 0) break; @@ -97,8 +97,8 @@ bool GCManager::work() } if (iter == storages.end()) iter = storages.begin(); - next_table_id = iter->first; - LOG_DEBUG(log, "End GC and next gc will start with table id: {}", next_table_id); + next_keyspace_table_id = iter->first; + LOG_DEBUG(log, "End GC and next gc will start with keyspace {}, table id: {}", next_keyspace_table_id.first, next_keyspace_table_id.second); gc_check_stop_watch.restart(); // Always return false return false; diff --git a/dbms/src/Storages/GCManager.h b/dbms/src/Storages/GCManager.h index 6f9aaf73187..0a28718f276 100644 --- a/dbms/src/Storages/GCManager.h +++ b/dbms/src/Storages/GCManager.h @@ -38,7 +38,7 @@ class GCManager private: Context & global_context; - TableID next_table_id = InvalidTableID; + KeyspaceTableID next_keyspace_table_id = KeyspaceTableID{NullspaceID, InvalidTableID}; AtomicStopwatch gc_check_stop_watch; diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index 93b58916443..9dd9ab51885 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -1516,8 +1516,10 @@ void StorageDeltaMerge::removeFromTMTContext() { // remove this table from TMTContext TMTContext & tmt_context = global_context.getTMTContext(); - tmt_context.getStorages().remove(tidb_table_info.id); - tmt_context.getRegionTable().removeTable(tidb_table_info.id); + auto keyspace_id = tidb_table_info.keyspace_id; + auto table_id = tidb_table_info.id; + tmt_context.getStorages().remove(keyspace_id, table_id); + tmt_context.getRegionTable().removeTable(keyspace_id, table_id); } StorageDeltaMerge::~StorageDeltaMerge() diff --git a/dbms/src/Storages/Transaction/ApplySnapshot.cpp b/dbms/src/Storages/Transaction/ApplySnapshot.cpp index 28fb0d307ce..c1e78a6bc58 100644 --- a/dbms/src/Storages/Transaction/ApplySnapshot.cpp +++ b/dbms/src/Storages/Transaction/ApplySnapshot.cpp @@ -120,7 +120,8 @@ void KVStore::checkAndApplyPreHandledSnapshot(const RegionPtrWrap & new_region, { auto table_id = new_region->getMappedTableID(); - if (auto storage = tmt.getStorages().get(table_id); storage) + auto keyspace_id = new_region->getKeyspaceID(); + if (auto storage = tmt.getStorages().get(keyspace_id, table_id); storage) { switch (storage->engineType()) { @@ -143,10 +144,11 @@ template void KVStore::onSnapshot(const RegionPtrWrap & new_region_wrap, RegionPtr old_region, UInt64 old_region_index, TMTContext & tmt) { RegionID region_id = new_region_wrap->id(); + auto keyspace_id = new_region_wrap->getKeyspaceID(); { auto table_id = new_region_wrap->getMappedTableID(); - if (auto storage = tmt.getStorages().get(table_id); storage && storage->engineType() == TiDB::StorageEngine::DT) + if (auto storage = tmt.getStorages().get(keyspace_id, table_id); storage && storage->engineType() == TiDB::StorageEngine::DT) { try { @@ -293,6 +295,7 @@ std::vector KVStore::preHandleSSTsToDTFiles( TMTContext & tmt) { auto context = tmt.getContext(); + auto keyspace_id = new_region->getKeyspaceID(); bool force_decode = false; size_t expected_block_size = DEFAULT_MERGE_BLOCK_SIZE; @@ -385,7 +388,7 @@ std::vector KVStore::preHandleSSTsToDTFiles( // Update schema and try to decode again LOG_INFO(log, "Decoding Region snapshot data meet error, sync schema and try to decode again {} [error={}]", new_region->toString(true), e.displayText()); GET_METRIC(tiflash_schema_trigger_count, type_raft_decode).Increment(); - tmt.getSchemaSyncer()->syncSchemas(context); + tmt.getSchemaSyncer()->syncSchemas(context, keyspace_id); // Next time should force_decode force_decode = true; @@ -575,7 +578,8 @@ RegionPtr KVStore::handleIngestSSTByDTFile(const RegionPtr & region, const SSTVi if (!external_files.empty()) { auto table_id = region->getMappedTableID(); - if (auto storage = tmt.getStorages().get(table_id); storage) + auto keyspace_id = region->getKeyspaceID(); + if (auto storage = tmt.getStorages().get(keyspace_id, table_id); storage) { // Ingest DTFiles into DeltaMerge storage auto & context = tmt.getContext(); diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 4fe68e6086e..45f07744009 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -137,8 +137,9 @@ void KVStore::traverseRegions(std::function & bool KVStore::tryFlushRegionCacheInStorage(TMTContext & tmt, const Region & region, const LoggerPtr & log, bool try_until_succeed) { fiu_do_on(FailPoints::force_fail_in_flush_region_data, { return false; }); + auto keyspace_id = region.getKeyspaceID(); auto table_id = region.getMappedTableID(); - auto storage = tmt.getStorages().get(table_id); + auto storage = tmt.getStorages().get(keyspace_id, table_id); if (unlikely(storage == nullptr)) { LOG_WARNING(log, diff --git a/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp b/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp new file mode 100644 index 00000000000..3ab93dd1ed9 --- /dev/null +++ b/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp @@ -0,0 +1,47 @@ +#include +#include + +namespace DB +{ +KeyspaceSnapshot::KeyspaceSnapshot(KeyspaceID keyspace_id_, pingcap::kv::Cluster * cluster_, UInt64 version_) + : snap(cluster_, version_) +{ + if (keyspace_id_ == NullspaceID) + return; + prefix = std::string(KEYSPACE_PREFIX_LEN, 0); + auto id = toBigEndian(keyspace_id_); + memcpy(prefix.data(), reinterpret_cast(&id), sizeof(KeyspaceID)); + prefix[0] = DB::TXN_MODE_PREFIX; +} + +std::string KeyspaceSnapshot::Get(const std::string & key) +{ + auto encoded_key = encodeKey(key); + return snap.Get(encoded_key); +} + +std::string KeyspaceSnapshot::Get(pingcap::kv::Backoffer & bo, const std::string & key) +{ + auto encoded_key = encodeKey(key); + return snap.Get(bo, encoded_key); +} + +KeyspaceScanner KeyspaceSnapshot::Scan(const std::string & begin, const std::string & end) +{ + auto inner = snap.Scan(encodeKey(begin), encodeKey(end)); + return KeyspaceScanner(inner, !prefix.empty()); +} + +std::string KeyspaceSnapshot::encodeKey(const std::string & key) +{ + return prefix.empty() ? key : prefix + key; +} + +std::string KeyspaceScanner::key() +{ + auto k = Base::key(); + if (need_cut) + k = k.substr(DB::KEYSPACE_PREFIX_LEN); + return k; +} +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/Transaction/KeyspaceSnapshot.h b/dbms/src/Storages/Transaction/KeyspaceSnapshot.h new file mode 100644 index 00000000000..d56dbd675e3 --- /dev/null +++ b/dbms/src/Storages/Transaction/KeyspaceSnapshot.h @@ -0,0 +1,49 @@ +#pragma once + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +#include +#pragma GCC diagnostic pop + +#include +#include +#include + +namespace DB +{ +struct KeyspaceScanner : public pingcap::kv::Scanner +{ + using Base = pingcap::kv::Scanner; + + KeyspaceScanner(Base scanner_, bool need_cut_) + : Base(scanner_) + , need_cut(need_cut_) + { + } + + std::string key(); + +private: + bool need_cut; +}; + +class KeyspaceSnapshot +{ +public: + using Base = pingcap::kv::Snapshot; + explicit KeyspaceSnapshot(KeyspaceID keyspace_id_, pingcap::kv::Cluster * cluster_, UInt64 version_); + + std::string Get(const std::string & key); + std::string Get(pingcap::kv::Backoffer & bo, const std::string & key); + KeyspaceScanner Scan(const std::string & begin, const std::string & end); + +private: + Base snap; + std::string prefix; + std::string encodeKey(const std::string & key); +}; +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/Transaction/LearnerRead.cpp b/dbms/src/Storages/Transaction/LearnerRead.cpp index 1cef544ccc8..98d64b4b58a 100644 --- a/dbms/src/Storages/Transaction/LearnerRead.cpp +++ b/dbms/src/Storages/Transaction/LearnerRead.cpp @@ -128,7 +128,7 @@ class MvccQueryInfoWrap regions_info_ptr = &*regions_info; // Only for test, because regions_query_info should never be empty if query is from TiDB or TiSpark. // todo support partition table - auto regions = tmt.getRegionTable().getRegionsByTable(logical_table_id); + auto regions = tmt.getRegionTable().getRegionsByTable(NullspaceID, logical_table_id); regions_info_ptr->reserve(regions.size()); for (const auto & [id, region] : regions) { diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 180a9dbc038..fddc5bddc82 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -61,13 +61,14 @@ static void writeRegionDataToStorage( { constexpr auto FUNCTION_NAME = __FUNCTION__; // NOLINT(readability-identifier-naming) const auto & tmt = context.getTMTContext(); + auto keyspace_id = region->getKeyspaceID(); TableID table_id = region->getMappedTableID(); UInt64 region_decode_cost = -1, write_part_cost = -1; /// Declare lambda of atomic read then write to call multiple times. auto atomic_read_write = [&](bool force_decode) { /// Get storage based on table ID. - auto storage = tmt.getStorages().get(table_id); + auto storage = tmt.getStorages().get(keyspace_id, table_id); if (storage == nullptr || storage->isTombstone()) { if (!force_decode) // Need to update. @@ -187,7 +188,7 @@ static void writeRegionDataToStorage( /// If first try failed, sync schema and force read then write. { GET_METRIC(tiflash_schema_trigger_count, type_raft_decode).Increment(); - tmt.getSchemaSyncer()->syncSchemas(context); + tmt.getSchemaSyncer()->syncSchemas(context, keyspace_id); if (!atomic_read_write(true)) { @@ -399,6 +400,7 @@ RegionTable::ResolveLocksAndWriteRegionRes RegionTable::resolveLocksAndWriteRegi /// Pre-decode region data into block cache and remove committed data from `region` RegionPtrWithBlock::CachePtr GenRegionPreDecodeBlockData(const RegionPtr & region, Context & context) { + auto keyspace_id = region->getKeyspaceID(); const auto & tmt = context.getTMTContext(); { Timestamp gc_safe_point = 0; @@ -441,7 +443,7 @@ RegionPtrWithBlock::CachePtr GenRegionPreDecodeBlockData(const RegionPtr & regio const auto atomic_decode = [&](bool force_decode) -> bool { Stopwatch watch; - auto storage = tmt.getStorages().get(table_id); + auto storage = tmt.getStorages().get(keyspace_id, table_id); if (storage == nullptr || storage->isTombstone()) { if (!force_decode) // Need to update. @@ -483,7 +485,7 @@ RegionPtrWithBlock::CachePtr GenRegionPreDecodeBlockData(const RegionPtr & regio if (!atomic_decode(false)) { GET_METRIC(tiflash_schema_trigger_count, type_raft_decode).Increment(); - tmt.getSchemaSyncer()->syncSchemas(context); + tmt.getSchemaSyncer()->syncSchemas(context, keyspace_id); if (!atomic_decode(true)) throw Exception("Pre-decode " + region->toString() + " cache to table " + std::to_string(table_id) + " block failed", @@ -502,11 +504,12 @@ AtomicGetStorageSchema(const RegionPtr & region, TMTContext & tmt) std::shared_ptr dm_storage; DecodingStorageSchemaSnapshotConstPtr schema_snapshot; + auto keyspace_id = region->getKeyspaceID(); auto table_id = region->getMappedTableID(); LOG_DEBUG(Logger::get(__PRETTY_FUNCTION__), "Get schema for table {}", table_id); auto context = tmt.getContext(); const auto atomic_get = [&](bool force_decode) -> bool { - auto storage = tmt.getStorages().get(table_id); + auto storage = tmt.getStorages().get(keyspace_id, table_id); if (storage == nullptr) { if (!force_decode) @@ -527,7 +530,7 @@ AtomicGetStorageSchema(const RegionPtr & region, TMTContext & tmt) if (!atomic_get(false)) { GET_METRIC(tiflash_schema_trigger_count, type_raft_decode).Increment(); - tmt.getSchemaSyncer()->syncSchemas(context); + tmt.getSchemaSyncer()->syncSchemas(context, keyspace_id); if (!atomic_get(true)) throw Exception("Get " + region->toString() + " belonging table " + DB::toString(table_id) + " is_command_handle fail", diff --git a/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp b/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp index f3551bed94e..cb8179a535d 100644 --- a/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp +++ b/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp @@ -20,6 +20,8 @@ #include #include +#include + namespace DB { HttpRequestRes HandleHttpRequestSyncStatus( @@ -31,11 +33,26 @@ HttpRequestRes HandleHttpRequestSyncStatus( { HttpRequestStatus status = HttpRequestStatus::Ok; TableID table_id = 0; + pingcap::pd::KeyspaceID keyspace_id = NullspaceID; { - std::string table_id_str(path.substr(api_name.size())); + auto * log = &Poco::Logger::get("HandleHttpRequestSyncStatus"); + LOG_DEBUG(log, "handling sync status request, path: {}, api_name: {}", path, api_name); + + // Query schema: /keyspace/{keyspace_id}/table/{table_id} + auto query = path.substr(api_name.size()); + std::vector query_parts; + boost::split(query_parts, query, boost::is_any_of("/")); + if (query_parts.size() < 4 || query_parts[0] != "keyspace" || query_parts[2] != "table") + { + LOG_ERROR(log, "invalid SyncStatus request: {}", query); + status = HttpRequestStatus::ErrorParam; + return HttpRequestRes{.status = status, .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{}}}; + } + try { - table_id = std::stoll(table_id_str); + keyspace_id = std::stoll(query_parts[1]); + table_id = std::stoll(query_parts[3]); } catch (...) { @@ -57,10 +74,11 @@ HttpRequestRes HandleHttpRequestSyncStatus( static const std::chrono::minutes PRINT_LOG_INTERVAL = std::chrono::minutes{5}; static Timepoint last_print_log_time = Clock::now(); // if storage is not created in ch, flash replica should not be available. - if (tmt.getStorages().get(table_id)) + // TODO(iosmanthus): TiDB should support tiflash replica. + if (tmt.getStorages().get(keyspace_id, table_id)) { RegionTable & region_table = tmt.getRegionTable(); - region_table.handleInternalRegionsByTable(table_id, [&](const RegionTable::InternalRegions & regions) { + region_table.handleInternalRegionsByTable(keyspace_id, table_id, [&](const RegionTable::InternalRegions & regions) { region_list.reserve(regions.size()); bool can_log = Clock::now() > last_print_log_time + PRINT_LOG_INTERVAL; FmtBuffer lag_regions_log; diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index 56134925ba6..7fee344df79 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -778,6 +778,7 @@ Region::Region(DB::RegionMeta && meta_, const TiFlashRaftProxyHelper * proxy_hel : meta(std::move(meta_)) , log(Logger::get()) , mapped_table_id(meta.getRange()->getMappedTableID()) + , keyspace_id(meta.getRange()->getKeyspaceID()) , proxy_helper(proxy_helper_) {} @@ -786,6 +787,11 @@ TableID Region::getMappedTableID() const return mapped_table_id; } +KeyspaceID Region::getKeyspaceID() const +{ + return keyspace_id; +} + void Region::setPeerState(raft_serverpb::PeerState state) { meta.setPeerState(state); diff --git a/dbms/src/Storages/Transaction/Region.h b/dbms/src/Storages/Transaction/Region.h index f4aead264c9..869a2ca44fe 100644 --- a/dbms/src/Storages/Transaction/Region.h +++ b/dbms/src/Storages/Transaction/Region.h @@ -193,6 +193,7 @@ class Region : public std::enable_shared_from_this raft_serverpb::MergeState getMergeState() const; TableID getMappedTableID() const; + KeyspaceID getKeyspaceID() const; EngineStoreApplyRes handleWriteRaftCmd(const WriteCmdsView & cmds, UInt64 index, UInt64 term, TMTContext & tmt); void finishIngestSSTByDTFile(RegionPtr && rhs, UInt64 index, UInt64 term); @@ -231,6 +232,7 @@ class Region : public std::enable_shared_from_this LoggerPtr log; const TableID mapped_table_id; + const KeyspaceID keyspace_id; std::atomic snapshot_event_flag{1}; const TiFlashRaftProxyHelper * proxy_helper{nullptr}; diff --git a/dbms/src/Storages/Transaction/RegionRangeKeys.h b/dbms/src/Storages/Transaction/RegionRangeKeys.h index 4e3c39d3765..a69bdb6ccb3 100644 --- a/dbms/src/Storages/Transaction/RegionRangeKeys.h +++ b/dbms/src/Storages/Transaction/RegionRangeKeys.h @@ -56,11 +56,13 @@ class RegionRangeKeys : boost::noncopyable const std::pair & rawKeys() const; explicit RegionRangeKeys(TiKVKey && start_key, TiKVKey && end_key); TableID getMappedTableID() const; + KeyspaceID getKeyspaceID() const; private: RegionRange ori; std::pair raw; TableID mapped_table_id; + KeyspaceID keyspace_id; }; } // namespace DB diff --git a/dbms/src/Storages/Transaction/RegionState.cpp b/dbms/src/Storages/Transaction/RegionState.cpp index cae75518709..bcc5444b5bd 100644 --- a/dbms/src/Storages/Transaction/RegionState.cpp +++ b/dbms/src/Storages/Transaction/RegionState.cpp @@ -122,11 +122,12 @@ raft_serverpb::MergeState & RegionState::getMutMergeState() bool computeMappedTableID(const DecodedTiKVKey & key, TableID & table_id) { + auto k = key.getUserKey(); // t table_id _r - if (key.size() >= (1 + 8 + 2) && key[0] == RecordKVFormat::TABLE_PREFIX - && memcmp(key.data() + 9, RecordKVFormat::RECORD_PREFIX_SEP, 2) == 0) + if (k.size() >= (1 + 8 + 2) && k[0] == RecordKVFormat::TABLE_PREFIX + && memcmp(k.data() + 9, RecordKVFormat::RECORD_PREFIX_SEP, 2) == 0) { - table_id = RecordKVFormat::getTableId(key); + table_id = RecordKVFormat::getTableId(k); return true; } @@ -138,6 +139,7 @@ RegionRangeKeys::RegionRangeKeys(TiKVKey && start_key, TiKVKey && end_key) , raw(std::make_shared(ori.first.key.empty() ? DecodedTiKVKey() : RecordKVFormat::decodeTiKVKey(ori.first.key)), std::make_shared(ori.second.key.empty() ? DecodedTiKVKey() : RecordKVFormat::decodeTiKVKey(ori.second.key))) { + keyspace_id = raw.first->getKeyspaceID(); if (!computeMappedTableID(*raw.first, mapped_table_id) || ori.first.compare(ori.second) >= 0) { throw Exception("Illegal region range, should not happen, start key: " + ori.first.key.toDebugString() @@ -151,6 +153,11 @@ TableID RegionRangeKeys::getMappedTableID() const return mapped_table_id; } +KeyspaceID RegionRangeKeys::getKeyspaceID() const +{ + return keyspace_id; +} + const std::pair & RegionRangeKeys::rawKeys() const { return raw; diff --git a/dbms/src/Storages/Transaction/RegionTable.cpp b/dbms/src/Storages/Transaction/RegionTable.cpp index dbca2f115a2..289e67bf83b 100644 --- a/dbms/src/Storages/Transaction/RegionTable.cpp +++ b/dbms/src/Storages/Transaction/RegionTable.cpp @@ -35,13 +35,14 @@ extern const int ILLFORMAT_RAFT_ROW; extern const int TABLE_IS_DROPPED; } // namespace ErrorCodes -RegionTable::Table & RegionTable::getOrCreateTable(const TableID table_id) +RegionTable::Table & RegionTable::getOrCreateTable(const KeyspaceID keyspace_id, const TableID table_id) { - auto it = tables.find(table_id); + auto ks_tb_id = KeyspaceTableID{keyspace_id, table_id}; + auto it = tables.find(ks_tb_id); if (it == tables.end()) { // Load persisted info. - it = tables.emplace(table_id, table_id).first; + it = tables.emplace(ks_tb_id, table_id).first; LOG_INFO(log, "get new table {}", table_id); } return it->second; @@ -55,6 +56,7 @@ RegionTable::InternalRegion & RegionTable::insertRegion(Table & table, const Reg RegionTable::InternalRegion & RegionTable::insertRegion(Table & table, const RegionRangeKeys & region_range_keys, const RegionID region_id) { + auto keyspace_id = region_range_keys.getKeyspaceID(); auto & table_regions = table.regions; // Insert table mapping. // todo check if region_range_keys.mapped_table_id == table.table_id ?? @@ -65,20 +67,21 @@ RegionTable::InternalRegion & RegionTable::insertRegion(Table & table, const Reg ErrorCodes::LOGICAL_ERROR); // Insert region mapping. - regions[region_id] = table.table_id; + regions[region_id] = KeyspaceTableID{keyspace_id, table.table_id}; return it->second; } -RegionTable::InternalRegion & RegionTable::doGetInternalRegion(DB::TableID table_id, DB::RegionID region_id) +RegionTable::InternalRegion & RegionTable::doGetInternalRegion(KeyspaceTableID ks_tb_id, DB::RegionID region_id) { - return tables.find(table_id)->second.regions.find(region_id)->second; + return tables.find(ks_tb_id)->second.regions.find(region_id)->second; } RegionTable::InternalRegion & RegionTable::getOrInsertRegion(const Region & region) { + auto keyspace_id = region.getKeyspaceID(); auto table_id = region.getMappedTableID(); - auto & table = getOrCreateTable(table_id); + auto & table = getOrCreateTable(keyspace_id, table_id); auto & table_regions = table.regions; if (auto it = table_regions.find(region.id()); it != table_regions.end()) return it->second; @@ -175,11 +178,11 @@ void RegionTable::restore() LOG_INFO(log, "Restore {} tables", tables.size()); } -void RegionTable::removeTable(TableID table_id) +void RegionTable::removeTable(KeyspaceID keyspace_id, TableID table_id) { std::lock_guard lock(mutex); - auto it = tables.find(table_id); + auto it = tables.find(KeyspaceTableID{keyspace_id, table_id}); if (it == tables.end()) return; auto & table = it->second; @@ -215,11 +218,11 @@ namespace /// Note that this function will try to acquire lock by `IStorage->lockForShare` void removeObsoleteDataInStorage( Context * const context, - const TableID table_id, + const KeyspaceTableID ks_table_id, const std::pair & handle_range) { TMTContext & tmt = context->getTMTContext(); - auto storage = tmt.getStorages().get(table_id); + auto storage = tmt.getStorages().get(ks_table_id.first, ks_table_id.second); // For DT only now if (!storage || storage->engineType() != TiDB::StorageEngine::DT) return; @@ -235,7 +238,7 @@ void removeObsoleteDataInStorage( /// Now we assume that these won't block for long time. auto rowkey_range - = DM::RowKeyRange::fromRegionRange(handle_range, table_id, table_id, storage->isCommonHandle(), storage->getRowKeyColumnSize()); + = DM::RowKeyRange::fromRegionRange(handle_range, ks_table_id.second, ks_table_id.second, storage->isCommonHandle(), storage->getRowKeyColumnSize()); dm_storage->deleteRange(rowkey_range, context->getSettingsRef()); dm_storage->flushCache(*context, rowkey_range, /*try_until_succeed*/ true); // flush to disk } @@ -250,7 +253,7 @@ void removeObsoleteDataInStorage( void RegionTable::removeRegion(const RegionID region_id, bool remove_data, const RegionTaskLock &) { - TableID table_id = 0; + KeyspaceTableID ks_tb_id; std::pair handle_range; { @@ -264,8 +267,8 @@ void RegionTable::removeRegion(const RegionID region_id, bool remove_data, const return; } - table_id = it->second; - auto & table = tables.find(table_id)->second; + ks_tb_id = it->second; + auto & table = tables.find(ks_tb_id)->second; auto internal_region_it = table.regions.find(region_id); handle_range = internal_region_it->second.range_in_table; @@ -277,7 +280,7 @@ void RegionTable::removeRegion(const RegionID region_id, bool remove_data, const table.regions.erase(internal_region_it); if (table.regions.empty()) { - tables.erase(table_id); + tables.erase(ks_tb_id); } LOG_INFO(log, "remove [region {}] in RegionTable done", region_id); } @@ -291,7 +294,7 @@ void RegionTable::removeRegion(const RegionID region_id, bool remove_data, const // But caller(KVStore) should ensure that no new data write into this handle_range // before `removeObsoleteDataInStorage` is done. (by param `RegionTaskLock`) // And this is expected not to block for long time. - removeObsoleteDataInStorage(context, table_id, handle_range); + removeObsoleteDataInStorage(context, ks_tb_id, handle_range); LOG_INFO(log, "remove region [{}] in storage done", region_id); } } @@ -387,8 +390,7 @@ RegionID RegionTable::pickRegionToFlush() auto region_id = *dirty_it; if (auto it = regions.find(region_id); it != regions.end()) { - auto table_id = it->second; - if (shouldFlush(doGetInternalRegion(table_id, region_id))) + if (shouldFlush(doGetInternalRegion(it->second, region_id))) { // The dirty flag should only be removed after data is flush successfully. return region_id; @@ -416,19 +418,19 @@ bool RegionTable::tryFlushRegions() return false; } -void RegionTable::handleInternalRegionsByTable(const TableID table_id, std::function && callback) const +void RegionTable::handleInternalRegionsByTable(const KeyspaceID keyspace_id, const TableID table_id, std::function && callback) const { std::lock_guard lock(mutex); - if (auto it = tables.find(table_id); it != tables.end()) + if (auto it = tables.find(KeyspaceTableID{keyspace_id, table_id}); it != tables.end()) callback(it->second.regions); } -std::vector> RegionTable::getRegionsByTable(const TableID table_id) const +std::vector> RegionTable::getRegionsByTable(const KeyspaceID keyspace_id, const TableID table_id) const { auto & kvstore = context->getTMTContext().getKVStore(); std::vector> regions; - handleInternalRegionsByTable(table_id, [&](const InternalRegions & internal_regions) { + handleInternalRegionsByTable(keyspace_id, table_id, [&](const InternalRegions & internal_regions) { for (const auto & region_info : internal_regions) { auto region = kvstore->getRegion(region_info.first); @@ -448,17 +450,19 @@ void RegionTable::extendRegionRange(const RegionID region_id, const RegionRangeK { std::lock_guard lock(mutex); + auto keyspace_id = region_range_keys.getKeyspaceID(); auto table_id = region_range_keys.getMappedTableID(); + auto ks_tb_id = KeyspaceTableID{keyspace_id, table_id}; auto new_handle_range = region_range_keys.rawKeys(); if (auto it = regions.find(region_id); it != regions.end()) { - if (table_id != it->second) + if (ks_tb_id != it->second) throw Exception(std::string(__PRETTY_FUNCTION__) + ": table id " + std::to_string(table_id) + " not match previous one " - + std::to_string(it->second) + " in regions " + std::to_string(region_id), + + std::to_string(it->second.second) + " in regions " + std::to_string(region_id), ErrorCodes::LOGICAL_ERROR); - InternalRegion & internal_region = doGetInternalRegion(table_id, region_id); + InternalRegion & internal_region = doGetInternalRegion(ks_tb_id, region_id); if (*(internal_region.range_in_table.first) <= *(new_handle_range.first) && *(internal_region.range_in_table.second) >= *(new_handle_range.second)) { @@ -476,7 +480,7 @@ void RegionTable::extendRegionRange(const RegionID region_id, const RegionRangeK } else { - auto & table = getOrCreateTable(table_id); + auto & table = getOrCreateTable(keyspace_id, table_id); insertRegion(table, region_range_keys, region_id); LOG_INFO(log, "table {} insert internal region {}", table_id, region_id); } diff --git a/dbms/src/Storages/Transaction/RegionTable.h b/dbms/src/Storages/Transaction/RegionTable.h index 36686b44d90..03f60f1644f 100644 --- a/dbms/src/Storages/Transaction/RegionTable.h +++ b/dbms/src/Storages/Transaction/RegionTable.h @@ -96,8 +96,8 @@ class RegionTable : private boost::noncopyable InternalRegions regions; }; - using TableMap = std::unordered_map; - using RegionInfoMap = std::unordered_map; + using TableMap = std::unordered_map>; + using RegionInfoMap = std::unordered_map; // safe ts is maintained by check_leader RPC (https://github.com/tikv/tikv/blob/1ea26a2ac8761af356cc5c0825eb89a0b8fc9749/components/resolved_ts/src/advance.rs#L262), // leader_safe_ts is the safe_ts in leader, leader will send to learner to advance safe_ts of learner, and TiFlash will record the safe_ts into safe_ts_map in check_leader RPC. @@ -161,8 +161,8 @@ class RegionTable : private boost::noncopyable RegionDataReadInfoList tryFlushRegion(RegionID region_id, bool try_persist = false); RegionDataReadInfoList tryFlushRegion(const RegionPtrWithBlock & region, bool try_persist); - void handleInternalRegionsByTable(TableID table_id, std::function && callback) const; - std::vector> getRegionsByTable(TableID table_id) const; + void handleInternalRegionsByTable(KeyspaceID keyspace_id, TableID table_id, std::function && callback) const; + std::vector> getRegionsByTable(KeyspaceID keyspace_id, TableID table_id) const; /// Write the data of the given region into the table with the given table ID, fill the data list for outer to remove. /// Will trigger schema sync on read error for only once, @@ -202,12 +202,12 @@ class RegionTable : private boost::noncopyable friend class MockTiDB; friend class StorageDeltaMerge; - Table & getOrCreateTable(TableID table_id); - void removeTable(TableID table_id); + Table & getOrCreateTable(KeyspaceID keyspace_id, TableID table_id); + void removeTable(KeyspaceID keyspace_id, TableID table_id); InternalRegion & getOrInsertRegion(const Region & region); InternalRegion & insertRegion(Table & table, const RegionRangeKeys & region_range_keys, RegionID region_id); InternalRegion & insertRegion(Table & table, const Region & region); - InternalRegion & doGetInternalRegion(TableID table_id, RegionID region_id); + InternalRegion & doGetInternalRegion(KeyspaceTableID ks_tb_id, RegionID region_id); RegionDataReadInfoList flushRegion(const RegionPtrWithBlock & region, bool try_persist) const; bool shouldFlush(const InternalRegion & region) const; diff --git a/dbms/src/Storages/Transaction/TMTStorages.cpp b/dbms/src/Storages/Transaction/TMTStorages.cpp index e1b0ac0826d..5340f5cce8b 100644 --- a/dbms/src/Storages/Transaction/TMTStorages.cpp +++ b/dbms/src/Storages/Transaction/TMTStorages.cpp @@ -27,36 +27,46 @@ void ManagedStorages::put(ManageableStoragePtr storage) { std::lock_guard lock(mutex); + KeyspaceID keyspace_id = storage->getTableInfo().keyspace_id; TableID table_id = storage->getTableInfo().id; - if (storages.find(table_id) != storages.end() && table_id != DB::InvalidTableID) + auto keyspace_table_id = KeyspaceTableID{keyspace_id, table_id}; + if (storages.find(keyspace_table_id) != storages.end() && table_id != DB::InvalidTableID) { // If table already exists, and is not created through ch-client (which table_id could be unspecified) // throw Exception throw Exception("TiDB table with id " + DB::toString(table_id) + " already exists.", ErrorCodes::TIDB_TABLE_ALREADY_EXISTS); } - storages.emplace(table_id, storage); + storages.emplace(keyspace_table_id, storage); + auto [it, _] = keyspaces.try_emplace(keyspace_id, 0); + it->second++; } -ManageableStoragePtr ManagedStorages::get(TableID table_id) const +ManageableStoragePtr ManagedStorages::get(KeyspaceID keyspace_id, TableID table_id) const { std::lock_guard lock(mutex); - if (auto it = storages.find(table_id); it != storages.end()) + if (auto it = storages.find(KeyspaceTableID{keyspace_id, table_id}); it != storages.end()) return it->second; return nullptr; } -std::unordered_map ManagedStorages::getAllStorage() const +StorageMap ManagedStorages::getAllStorage() const { std::lock_guard lock(mutex); return storages; } +KeyspaceSet ManagedStorages::getAllKeyspaces() const +{ + std::lock_guard lock(mutex); + return keyspaces; +} + ManageableStoragePtr ManagedStorages::getByName(const std::string & db, const std::string & table, bool include_tombstone) const { std::lock_guard lock(mutex); - auto it = std::find_if(storages.begin(), storages.end(), [&](const std::pair & pair) { + auto it = std::find_if(storages.begin(), storages.end(), [&](const std::pair & pair) { const auto & storage = pair.second; return (include_tombstone || !storage->isTombstone()) && storage->getDatabaseName() == db && storage->getTableInfo().name == table; }); @@ -65,14 +75,17 @@ ManageableStoragePtr ManagedStorages::getByName(const std::string & db, const st return it->second; } -void ManagedStorages::remove(TableID table_id) +void ManagedStorages::remove(KeyspaceID keyspace_id, TableID table_id) { std::lock_guard lock(mutex); - auto it = storages.find(table_id); + auto it = storages.find(KeyspaceTableID{keyspace_id, table_id}); if (it == storages.end()) return; storages.erase(it); + keyspaces[keyspace_id]--; + if (!keyspaces[keyspace_id]) + keyspaces.erase(keyspace_id); } } // namespace DB diff --git a/dbms/src/Storages/Transaction/TMTStorages.h b/dbms/src/Storages/Transaction/TMTStorages.h index 9df36caa999..c5be3f8f777 100644 --- a/dbms/src/Storages/Transaction/TMTStorages.h +++ b/dbms/src/Storages/Transaction/TMTStorages.h @@ -26,21 +26,25 @@ class IManageableStorage; class StorageDeltaMerge; using StorageDeltaMergePtr = std::shared_ptr; using ManageableStoragePtr = std::shared_ptr; +using StorageMap = std::unordered_map>; +using KeyspaceSet = std::unordered_map; class ManagedStorages : private boost::noncopyable { public: void put(ManageableStoragePtr storage); - ManageableStoragePtr get(TableID table_id) const; - std::unordered_map getAllStorage() const; + ManageableStoragePtr get(KeyspaceID keyspace_id, TableID table_id) const; + StorageMap getAllStorage() const; + KeyspaceSet getAllKeyspaces() const; ManageableStoragePtr getByName(const std::string & db, const std::string & table, bool include_tombstone) const; - void remove(TableID table_id); + void remove(KeyspaceID keyspace_id, TableID table_id); private: - std::unordered_map storages; + StorageMap storages; + KeyspaceSet keyspaces; mutable std::mutex mutex; }; diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 40f189d9009..32b076679b5 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -588,6 +588,7 @@ try Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); json->set("id", id); + json->set("keyspace_id", keyspace_id); Poco::JSON::Object::Ptr name_json = new Poco::JSON::Object(); name_json->set("O", name); name_json->set("L", name); @@ -616,6 +617,10 @@ try Poco::Dynamic::Var result = parser.parse(json_str); auto obj = result.extract(); id = obj->getValue("id"); + if (obj->has("keyspace_id")) + { + keyspace_id = obj->getValue("keyspace_id"); + } name = obj->get("db_name").extract()->get("L").convert(); charset = obj->get("charset").convert(); collate = obj->get("collate").convert(); @@ -777,14 +782,23 @@ catch (const Poco::Exception & e) /////////////////////// ////// TableInfo ////// /////////////////////// -TableInfo::TableInfo(Poco::JSON::Object::Ptr json) +TableInfo::TableInfo(Poco::JSON::Object::Ptr json, KeyspaceID keyspace_id_) { deserialize(json); + if (keyspace_id == NullspaceID) + { + keyspace_id = keyspace_id_; + } } -TableInfo::TableInfo(const String & table_info_json) +TableInfo::TableInfo(const String & table_info_json, KeyspaceID keyspace_id_) { deserialize(table_info_json); + // If the table_info_json has no keyspace id, we use the keyspace_id_ as the default value. + if (keyspace_id == NullspaceID) + { + keyspace_id = keyspace_id_; + } } String TableInfo::serialize() const @@ -794,6 +808,7 @@ try Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); json->set("id", id); + json->set("keyspace_id", keyspace_id); Poco::JSON::Object::Ptr name_json = new Poco::JSON::Object(); name_json->set("O", name); name_json->set("L", name); @@ -864,6 +879,10 @@ void TableInfo::deserialize(Poco::JSON::Object::Ptr obj) try { id = obj->getValue("id"); + if (obj->has("keyspace_id")) + { + keyspace_id = obj->getValue("keyspace_id"); + } name = obj->getObject("name")->getValue("L"); auto cols_arr = obj->getArray("cols"); diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index c41a2f5157e..f3caf734285 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -50,6 +50,8 @@ using DB::DatabaseID; using DB::String; using DB::TableID; using DB::Timestamp; +using DB::KeyspaceID; +using DB::NullspaceID; // Column types. // In format: @@ -273,13 +275,21 @@ struct PartitionInfo struct DBInfo { DatabaseID id = -1; + KeyspaceID keyspace_id = NullspaceID; String name; String charset; String collate; SchemaState state; DBInfo() = default; - explicit DBInfo(const String & json) { deserialize(json); } + explicit DBInfo(const String & json, KeyspaceID keyspace_id_) + { + deserialize(json); + if (keyspace_id == NullspaceID) + { + keyspace_id = keyspace_id_; + } + } String serialize() const; @@ -351,9 +361,9 @@ struct TableInfo TableInfo & operator=(const TableInfo &) = default; - explicit TableInfo(Poco::JSON::Object::Ptr json); + explicit TableInfo(Poco::JSON::Object::Ptr json, KeyspaceID keyspace_id_); - explicit TableInfo(const String & table_info_json); + explicit TableInfo(const String & table_info_json, KeyspaceID keyspace_id_); String serialize() const; @@ -366,6 +376,8 @@ struct TableInfo // and partition ID for partition table, // whereas field `belonging_table_id` below actually means the table ID this partition belongs to. TableID id = DB::InvalidTableID; + // The keyspace where the table belongs to. + KeyspaceID keyspace_id = NullspaceID; String name; // Columns are listed in the order in which they appear in the schema. std::vector columns; diff --git a/dbms/src/Storages/Transaction/TiKVKeyValue.cpp b/dbms/src/Storages/Transaction/TiKVKeyValue.cpp new file mode 100644 index 00000000000..4829aca7fa1 --- /dev/null +++ b/dbms/src/Storages/Transaction/TiKVKeyValue.cpp @@ -0,0 +1,49 @@ +// Copyright 2023 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +KeyspaceID DecodedTiKVKey::getKeyspaceID() const +{ + if (size() < KEYSPACE_PREFIX_LEN || *begin() != TXN_MODE_PREFIX) + return NullspaceID; + + char buf[KEYSPACE_PREFIX_LEN]; + memcpy(buf, data(), KEYSPACE_PREFIX_LEN); + buf[0] = 0; + return toBigEndian(*reinterpret_cast(buf)); +} + +std::string_view DecodedTiKVKey::getUserKey() const +{ + if (size() < KEYSPACE_PREFIX_LEN || *begin() != TXN_MODE_PREFIX) + return std::string_view(c_str(), size()); + + return std::string_view(c_str() + KEYSPACE_PREFIX_LEN, size() - KEYSPACE_PREFIX_LEN); +} + +std::string DecodedTiKVKey::makeKeyspacePrefix(KeyspaceID keyspace_id) +{ + if (keyspace_id == NullspaceID) + return std::string(); + std::string prefix(KEYSPACE_PREFIX_LEN, 0); + keyspace_id = toBigEndian(keyspace_id); + memcpy(prefix.data(), reinterpret_cast(&keyspace_id), KEYSPACE_PREFIX_LEN); + prefix[0] = TXN_MODE_PREFIX; + return prefix; +} +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/Transaction/TiKVKeyValue.h b/dbms/src/Storages/Transaction/TiKVKeyValue.h index 45db99b03a4..5b86d2219d0 100644 --- a/dbms/src/Storages/Transaction/TiKVKeyValue.h +++ b/dbms/src/Storages/Transaction/TiKVKeyValue.h @@ -21,6 +21,9 @@ namespace DB { +static const size_t KEYSPACE_PREFIX_LEN = 4; +static const char TXN_MODE_PREFIX = 'x'; + template struct StringObject : std::string { @@ -100,6 +103,10 @@ struct DecodedTiKVKey : std::string (Base &)* this = (Base &&) obj; return *this; } + + KeyspaceID getKeyspaceID() const; + std::string_view getUserKey() const; + static std::string makeKeyspacePrefix(KeyspaceID keyspace_id); }; static_assert(sizeof(DecodedTiKVKey) == sizeof(std::string)); diff --git a/dbms/src/Storages/Transaction/TiKVRecordFormat.h b/dbms/src/Storages/Transaction/TiKVRecordFormat.h index 2c1739bb609..b4c296816fa 100644 --- a/dbms/src/Storages/Transaction/TiKVRecordFormat.h +++ b/dbms/src/Storages/Transaction/TiKVRecordFormat.h @@ -212,21 +212,29 @@ inline Timestamp getTs(const TiKVKey & key) return decodeUInt64Desc(read(key.data() + key.dataSize() - 8)); } -inline TableID getTableId(const DecodedTiKVKey & key) +template +inline TableID getTableId(const T & key) { return decodeInt64(read(key.data() + 1)); } inline HandleID getHandle(const DecodedTiKVKey & key) { - return decodeInt64(read(key.data() + RAW_KEY_NO_HANDLE_SIZE)); + return decodeInt64(read(key.getUserKey().data())); +} + +inline std::string_view getRawTiDBPKView(const DecodedTiKVKey & key) +{ + auto user_key = key.getUserKey(); + return std::string_view(user_key.data() + RAW_KEY_NO_HANDLE_SIZE, user_key.size() - RAW_KEY_NO_HANDLE_SIZE); } inline RawTiDBPK getRawTiDBPK(const DecodedTiKVKey & key) { - return std::make_shared(key.begin() + RAW_KEY_NO_HANDLE_SIZE, key.end()); + return std::make_shared(getRawTiDBPKView(key)); } + inline TableID getTableId(const TiKVKey & key) { return getTableId(decodeTiKVKey(key)); diff --git a/dbms/src/Storages/Transaction/Types.h b/dbms/src/Storages/Transaction/Types.h index 59622321510..08a04964927 100644 --- a/dbms/src/Storages/Transaction/Types.h +++ b/dbms/src/Storages/Transaction/Types.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include @@ -23,6 +24,11 @@ namespace DB { using TableID = Int64; using TableIDSet = std::unordered_set; +using KeyspaceID = pingcap::pd::KeyspaceID; + +using KeyspaceTableID = std::pair; + +static auto const NullspaceID = pingcap::pd::NullspaceID; enum : TableID { @@ -31,6 +37,8 @@ enum : TableID using DatabaseID = Int64; +using KeyspaceDatabaseID = std::pair; + using ColumnID = Int64; enum : ColumnID diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index cab0aa216f8..7d45460e093 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -403,7 +403,7 @@ void SchemaBuilder::applyAlterTable(const DBInfoPtr & db_inf throw TiFlashException(fmt::format("miss table in TiKV : {}", table_id), Errors::DDL::StaleSchema); } auto & tmt_context = context.getTMTContext(); - auto storage = tmt_context.getStorages().get(table_info->id); + auto storage = tmt_context.getStorages().get(keyspace_id, table_info->id); if (storage == nullptr) { throw TiFlashException(fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*db_info, *table_info)), @@ -427,7 +427,7 @@ void SchemaBuilder::applyAlterLogicalTable(const DBInfoPtr & for (const auto & part_def : table_info->partition.definitions) { auto part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper); - auto part_storage = tmt_context.getStorages().get(part_def.id); + auto part_storage = tmt_context.getStorages().get(keyspace_id, part_def.id); if (part_storage == nullptr) { throw TiFlashException(fmt::format("miss table in TiFlash : {}, partition: {}.", name_mapper.debugCanonicalName(*db_info, *table_info), part_def.id), @@ -589,7 +589,7 @@ void SchemaBuilder::applyPartitionDiff(const TiDB::DBInfoPtr } auto & tmt_context = context.getTMTContext(); - auto storage = tmt_context.getStorages().get(table_info->id); + auto storage = tmt_context.getStorages().get(keyspace_id, table_info->id); if (storage == nullptr) { throw TiFlashException(fmt::format("miss table in TiFlash {}", table_id), Errors::DDL::MissingTable); @@ -674,7 +674,7 @@ void SchemaBuilder::applyRenameTable(const DBInfoPtr & new_d } auto & tmt_context = context.getTMTContext(); - auto storage = tmt_context.getStorages().get(table_id); + auto storage = tmt_context.getStorages().get(keyspace_id, table_id); if (storage == nullptr) { throw TiFlashException(fmt::format("miss table id in TiFlash {}", table_id), Errors::DDL::MissingTable); @@ -696,7 +696,7 @@ void SchemaBuilder::applyRenameLogicalTable( auto & tmt_context = context.getTMTContext(); for (const auto & part_def : new_table_info->partition.definitions) { - auto part_storage = tmt_context.getStorages().get(part_def.id); + auto part_storage = tmt_context.getStorages().get(keyspace_id, part_def.id); if (part_storage == nullptr) { throw Exception(fmt::format("miss old table id in Flash {}", part_def.id)); @@ -783,7 +783,7 @@ void SchemaBuilder::applyExchangeTablePartition(const Schema if (table_info == nullptr) throw TiFlashException(fmt::format("miss table in TiKV : {}", pt_table_info), Errors::DDL::StaleSchema); auto & tmt_context = context.getTMTContext(); - auto storage = tmt_context.getStorages().get(table_info->id); + auto storage = tmt_context.getStorages().get(keyspace_id, table_info->id); if (storage == nullptr) throw TiFlashException( fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*pt_db_info, *table_info)), @@ -805,7 +805,7 @@ void SchemaBuilder::applyExchangeTablePartition(const Schema FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_after_step_1_in_exchange_partition); /// step 2 change non partition table to a partition of the partition table - storage = tmt_context.getStorages().get(npt_table_id); + storage = tmt_context.getStorages().get(keyspace_id, npt_table_id); if (storage == nullptr) throw TiFlashException(fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*npt_db_info, *table_info)), Errors::DDL::MissingTable); @@ -834,7 +834,7 @@ void SchemaBuilder::applyExchangeTablePartition(const Schema table_info = getter.getTableInfo(npt_db_info->id, pt_partition_id); if (table_info == nullptr) throw TiFlashException(fmt::format("miss table in TiKV : {}", pt_partition_id), Errors::DDL::StaleSchema); - storage = tmt_context.getStorages().get(table_info->id); + storage = tmt_context.getStorages().get(keyspace_id, table_info->id); if (storage == nullptr) throw TiFlashException( fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*pt_db_info, *table_info)), @@ -928,14 +928,15 @@ void SchemaBuilder::applyCreateSchema(const TiDB::DBInfoPtr interpreter.setForceRestoreData(false); interpreter.execute(); - databases[db_info->id] = db_info; + databases.emplace(KeyspaceDatabaseID{keyspace_id, db_info->id}, db_info); LOG_INFO(log, "Created database {}", name_mapper.debugDatabaseName(*db_info)); } template void SchemaBuilder::applyDropSchema(DatabaseID schema_id) { - auto it = databases.find(schema_id); + auto ks_db_id = KeyspaceDatabaseID{keyspace_id, schema_id}; + auto it = databases.find(ks_db_id); if (unlikely(it == databases.end())) { LOG_INFO( @@ -945,7 +946,7 @@ void SchemaBuilder::applyDropSchema(DatabaseID schema_id) return; } applyDropSchema(name_mapper.mapDatabaseName(*it->second)); - databases.erase(schema_id); + databases.erase(ks_db_id); } template @@ -1064,7 +1065,7 @@ void SchemaBuilder::applyCreatePhysicalTable(const DBInfoPtr /// Check if this is a RECOVER table. { auto & tmt_context = context.getTMTContext(); - if (auto * storage = tmt_context.getStorages().get(table_info->id).get(); storage) + if (auto * storage = tmt_context.getStorages().get(keyspace_id, table_info->id).get(); storage) { if (!storage->isTombstone()) { @@ -1149,7 +1150,7 @@ template void SchemaBuilder::applyDropPhysicalTable(const String & db_name, TableID table_id) { auto & tmt_context = context.getTMTContext(); - auto storage = tmt_context.getStorages().get(table_id); + auto storage = tmt_context.getStorages().get(keyspace_id, table_id); if (storage == nullptr) { LOG_DEBUG(log, "table {} does not exist.", table_id); @@ -1178,7 +1179,7 @@ template void SchemaBuilder::applyDropTable(const DBInfoPtr & db_info, TableID table_id) { auto & tmt_context = context.getTMTContext(); - auto * storage = tmt_context.getStorages().get(table_id).get(); + auto * storage = tmt_context.getStorages().get(keyspace_id, table_id).get(); if (storage == nullptr) { LOG_DEBUG(log, "table {} does not exist.", table_id); @@ -1208,7 +1209,7 @@ void SchemaBuilder::applySetTiFlashReplica(const TiDB::DBInf } auto & tmt_context = context.getTMTContext(); - auto storage = tmt_context.getStorages().get(latest_table_info->id); + auto storage = tmt_context.getStorages().get(keyspace_id, latest_table_info->id); if (unlikely(storage == nullptr)) { throw TiFlashException(fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*db_info, *latest_table_info)), @@ -1230,7 +1231,7 @@ void SchemaBuilder::applySetTiFlashReplicaOnLogicalTable(con for (const auto & part_def : table_info->partition.definitions) { auto new_part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper); - auto part_storage = tmt_context.getStorages().get(new_part_table_info->id); + auto part_storage = tmt_context.getStorages().get(keyspace_id, new_part_table_info->id); if (unlikely(part_storage == nullptr)) { throw TiFlashException(fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*db_info, *new_part_table_info)), @@ -1278,7 +1279,7 @@ void SchemaBuilder::syncAllSchema() for (const auto & db : all_schemas) { db_set.emplace(name_mapper.mapDatabaseName(*db)); - if (databases.find(db->id) == databases.end()) + if (databases.find(KeyspaceDatabaseID{keyspace_id, db->id}) == databases.end()) { applyCreateSchema(db); LOG_DEBUG(log, "Database {} created during sync all schemas", name_mapper.debugDatabaseName(*db)); @@ -1310,12 +1311,12 @@ void SchemaBuilder::syncAllSchema() }); } - auto storage = tmt_context.getStorages().get(table->id); + auto storage = tmt_context.getStorages().get(keyspace_id, table->id); if (storage == nullptr) { /// Create if not exists. applyCreateLogicalTable(db, table); - storage = tmt_context.getStorages().get(table->id); + storage = tmt_context.getStorages().get(keyspace_id, table->id); if (storage == nullptr) { /// This is abnormal as the storage shouldn't be null after creation, the underlying table must already be existing for unknown reason. @@ -1345,10 +1346,15 @@ void SchemaBuilder::syncAllSchema() auto storage_map = tmt_context.getStorages().getAllStorage(); for (auto it = storage_map.begin(); it != storage_map.end(); it++) { - if (table_set.count(it->first) == 0) + auto table_info = it->second->getTableInfo(); + if (table_info.keyspace_id != keyspace_id) { - applyDropPhysicalTable(it->second->getDatabaseName(), it->first); - LOG_DEBUG(log, "Table {}.{} dropped during sync all schemas", it->second->getDatabaseName(), name_mapper.debugTableName(it->second->getTableInfo())); + continue; + } + if (table_set.count(table_info.id) == 0) + { + applyDropPhysicalTable(it->second->getDatabaseName(), table_info.id); + LOG_DEBUG(log, "Table {}.{} dropped during sync all schemas", it->second->getDatabaseName(), name_mapper.debugTableName(table_info)); } } @@ -1356,6 +1362,11 @@ void SchemaBuilder::syncAllSchema() const auto & dbs = context.getDatabases(); for (auto it = dbs.begin(); it != dbs.end(); it++) { + auto db_keyspace_id = SchemaNameMapper::getMappedNameKeyspaceID(it->first); + if (db_keyspace_id != keyspace_id) + { + continue; + } if (db_set.count(it->first) == 0 && !isReservedDatabase(context, it->first)) { applyDropSchema(it->first); diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.h b/dbms/src/TiDB/Schema/SchemaBuilder.h index 461d7ff9c12..8837921ed52 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.h +++ b/dbms/src/TiDB/Schema/SchemaBuilder.h @@ -20,6 +20,7 @@ namespace DB { +using KeyspaceDatabaseMap = std::unordered_map>; template struct SchemaBuilder { @@ -29,17 +30,20 @@ struct SchemaBuilder Context & context; - std::unordered_map & databases; + KeyspaceDatabaseMap & databases; Int64 target_version; + KeyspaceID keyspace_id; + Poco::Logger * log; - SchemaBuilder(Getter & getter_, Context & context_, std::unordered_map & dbs_, Int64 version) + SchemaBuilder(Getter & getter_, Context & context_, KeyspaceDatabaseMap & dbs_, Int64 version) : getter(getter_) , context(context_) , databases(dbs_) , target_version(version) + , keyspace_id(getter_.getKeyspaceID()) , log(&Poco::Logger::get("SchemaBuilder")) {} diff --git a/dbms/src/TiDB/Schema/SchemaGetter.cpp b/dbms/src/TiDB/Schema/SchemaGetter.cpp index 9162e18dd75..f05e0ed8d47 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.cpp +++ b/dbms/src/TiDB/Schema/SchemaGetter.cpp @@ -15,7 +15,6 @@ #include #include #include -#include namespace DB { @@ -102,14 +101,14 @@ struct TxnStructure } public: - static String get(pingcap::kv::Snapshot & snap, const String & key) + static String get(KeyspaceSnapshot & snap, const String & key) { String encode_key = encodeStringDataKey(key); String value = snap.Get(encode_key); return value; } - static String hGet(pingcap::kv::Snapshot & snap, const String & key, const String & field) + static String hGet(KeyspaceSnapshot & snap, const String & key, const String & field) { String encode_key = encodeHashDataKey(key, field); String value = snap.Get(encode_key); @@ -117,7 +116,7 @@ struct TxnStructure } // For convinient, we only return values. - static std::vector> hGetAll(pingcap::kv::Snapshot & snap, const String & key) + static std::vector> hGetAll(KeyspaceSnapshot & snap, const String & key) { auto tikv_key_prefix = hashDataKeyPrefix(key); String tikv_key_end = pingcap::kv::prefixNext(tikv_key_prefix); @@ -249,7 +248,7 @@ TiDB::DBInfoPtr SchemaGetter::getDatabase(DatabaseID db_id) return nullptr; LOG_DEBUG(log, "Get DB Info from TiKV : " + json); - auto db_info = std::make_shared(json); + auto db_info = std::make_shared(json, keyspace_id); return db_info; } @@ -265,7 +264,7 @@ TiDB::TableInfoPtr SchemaGetter::getTableInfo(DatabaseID db_id, TableID table_id if (table_info_json.empty()) return nullptr; LOG_DEBUG(log, "Get Table Info from TiKV : " + table_info_json); - TiDB::TableInfoPtr table_info = std::make_shared(table_info_json); + TiDB::TableInfoPtr table_info = std::make_shared(table_info_json, keyspace_id); return table_info; } @@ -275,7 +274,7 @@ std::vector SchemaGetter::listDBs() auto pairs = TxnStructure::hGetAll(snap, DBs); for (const auto & pair : pairs) { - auto db_info = std::make_shared(pair.second); + auto db_info = std::make_shared(pair.second, keyspace_id); res.push_back(db_info); } return res; @@ -307,7 +306,7 @@ std::vector SchemaGetter::listTables(DatabaseID db_id) continue; } const String & json = kv_pair.second; - auto table_info = std::make_shared(json); + auto table_info = std::make_shared(json, keyspace_id); res.push_back(table_info); } diff --git a/dbms/src/TiDB/Schema/SchemaGetter.h b/dbms/src/TiDB/Schema/SchemaGetter.h index 45dd5d315e4..99b471c00d9 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.h +++ b/dbms/src/TiDB/Schema/SchemaGetter.h @@ -14,16 +14,8 @@ #pragma once +#include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" -#ifdef __clang__ -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif -#include -#pragma GCC diagnostic pop - #include #include @@ -137,14 +129,18 @@ struct SchemaDiff struct SchemaGetter { - pingcap::kv::Snapshot snap; + KeyspaceSnapshot snap; + + KeyspaceID keyspace_id; Poco::Logger * log; - SchemaGetter(pingcap::kv::Cluster * cluster_, UInt64 tso_) - : snap(cluster_, tso_) + SchemaGetter(pingcap::kv::Cluster * cluster_, UInt64 tso_, KeyspaceID keyspace_id_) + : snap(keyspace_id_, cluster_, tso_) + , keyspace_id(keyspace_id_) , log(&Poco::Logger::get("SchemaGetter")) - {} + { + } Int64 getVersion(); @@ -167,6 +163,8 @@ struct SchemaGetter std::vector listDBs(); std::vector listTables(DatabaseID db_id); + + KeyspaceID getKeyspaceID() const { return keyspace_id; } }; } // namespace DB diff --git a/dbms/src/TiDB/Schema/SchemaNameMapper.h b/dbms/src/TiDB/Schema/SchemaNameMapper.h index 12c014a9020..7fdebbb6f01 100644 --- a/dbms/src/TiDB/Schema/SchemaNameMapper.h +++ b/dbms/src/TiDB/Schema/SchemaNameMapper.h @@ -26,18 +26,56 @@ struct SchemaNameMapper static constexpr auto DATABASE_PREFIX = "db_"; static constexpr auto TABLE_PREFIX = "t_"; + static constexpr auto KEYSPACE_PREFIX = "ks_"; - virtual String mapDatabaseName(const TiDB::DBInfo & db_info) const { return DATABASE_PREFIX + std::to_string(db_info.id); } - virtual String displayDatabaseName(const TiDB::DBInfo & db_info) const { return db_info.name; } - virtual String mapTableName(const TiDB::TableInfo & table_info) const { return TABLE_PREFIX + std::to_string(table_info.id); } - virtual String displayTableName(const TiDB::TableInfo & table_info) const { return table_info.name; } + + static KeyspaceID getMappedNameKeyspaceID(const String & name) + { + auto keyspace_prefix_len = std::strlen(KEYSPACE_PREFIX); + auto pos = name.find(KEYSPACE_PREFIX); + if (pos == String::npos) + return NullspaceID; + assert(pos == 0); + pos = name.find('_', keyspace_prefix_len); + assert(pos != String::npos); + return std::stoull(name.substr(keyspace_prefix_len, pos - keyspace_prefix_len)); + } + + static String map2Keyspace(KeyspaceID keyspace_id, const String & name) + { + return keyspace_id == NullspaceID ? name : KEYSPACE_PREFIX + std::to_string(keyspace_id) + "_" + name; + } + + virtual String mapDatabaseName(const TiDB::DBInfo & db_info) const + { + auto db_name = DATABASE_PREFIX + std::to_string(db_info.id); + return map2Keyspace(db_info.keyspace_id, db_name); + } + virtual String displayDatabaseName(const TiDB::DBInfo & db_info) const + { + return map2Keyspace(db_info.keyspace_id, db_info.name); + } + virtual String mapTableName(const TiDB::TableInfo & table_info) const + { + auto table_name = TABLE_PREFIX + std::to_string(table_info.id); + return map2Keyspace(table_info.keyspace_id, table_name); + } + virtual String displayTableName(const TiDB::TableInfo & table_info) const + { + return map2Keyspace(table_info.keyspace_id, table_info.name); + } virtual String mapPartitionName(const TiDB::TableInfo & table_info) const { return mapTableName(table_info); } // Only use for logging / debugging - virtual String debugDatabaseName(const TiDB::DBInfo & db_info) const { return db_info.name + "(" + std::to_string(db_info.id) + ")"; } + virtual String debugDatabaseName(const TiDB::DBInfo & db_info) const + { + auto db_name = db_info.name + "(" + std::to_string(db_info.id) + ")"; + return map2Keyspace(db_info.keyspace_id, db_name); + } virtual String debugTableName(const TiDB::TableInfo & table_info) const { - return table_info.name + "(" + std::to_string(table_info.id) + ")"; + auto table_name = table_info.name + "(" + std::to_string(table_info.id) + ")"; + return map2Keyspace(table_info.keyspace_id, table_name); } virtual String debugCanonicalName(const TiDB::DBInfo & db_info, const TiDB::TableInfo & table_info) const { diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index 5027a769e08..d498bd4c85f 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -40,35 +40,42 @@ SchemaSyncService::SchemaSyncService(DB::Context & context_) handle = background_pool.addTask( [&, this] { String stage; + auto keyspaces = context.getTMTContext().getStorages().getAllKeyspaces(); + KeyspaceID cur_ks = NullspaceID; bool done_anything = false; try { - /// Do sync schema first, then gc. - /// They must be performed synchronously, - /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. - // GC safe point must be obtained ahead of syncing schema. auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient()); - stage = "Sync schemas"; - done_anything = syncSchemas(); - if (done_anything) - GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); - - stage = "GC"; - done_anything = gc(gc_safe_point); - + for (auto const iter : keyspaces) + { + auto ks = iter.first; + LOG_DEBUG(log, "Auto sync schema for keyspace {}", ks); + cur_ks = ks; + /// Do sync schema first, then gc. + /// They must be performed synchronously, + /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. + // GC safe point must be obtained ahead of syncing schema. + stage = "Sync schemas"; + done_anything = syncSchemas(ks); + if (done_anything) + GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); + + stage = "GC"; + done_anything = gc(gc_safe_point, ks); + } return done_anything; } catch (const Exception & e) { - LOG_ERROR(log, "{} failed by {} \n stack : {}", stage, e.displayText(), e.getStackTrace().toString()); + LOG_ERROR(log, "[keyspace {}] {} failed by {} \n stack : {}", cur_ks, stage, e.displayText(), e.getStackTrace().toString()); } catch (const Poco::Exception & e) { - LOG_ERROR(log, "{} failed by {}", stage, e.displayText()); + LOG_ERROR(log, "[keyspace {}] {} failed by {}", cur_ks, stage, e.displayText()); } catch (const std::exception & e) { - LOG_ERROR(log, "{} failed by {}", stage, e.what()); + LOG_ERROR(log, "[keyspace {}] {} failed by {}", cur_ks, stage, e.what()); } return false; }, @@ -80,9 +87,9 @@ SchemaSyncService::~SchemaSyncService() background_pool.removeTask(handle); } -bool SchemaSyncService::syncSchemas() +bool SchemaSyncService::syncSchemas(KeyspaceID keyspace_id) { - return context.getTMTContext().getSchemaSyncer()->syncSchemas(context); + return context.getTMTContext().getSchemaSyncer()->syncSchemas(context, keyspace_id); } template @@ -91,7 +98,7 @@ inline bool isSafeForGC(const DatabaseOrTablePtr & ptr, Timestamp gc_safe_point) return ptr->isTombstone() && ptr->getTombstone() < gc_safe_point; } -bool SchemaSyncService::gc(Timestamp gc_safe_point) +bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) { auto & tmt_context = context.getTMTContext(); if (gc_safe_point == gc_context.last_gc_safe_point) @@ -105,6 +112,9 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point) auto dbs = context.getDatabases(); for (const auto & iter : dbs) { + auto db_ks_id = SchemaNameMapper::getMappedNameKeyspaceID(iter.first); + if (db_ks_id != keyspace_id) + continue; const auto & db = iter.second; for (auto table_iter = db->getIterator(context); table_iter->isValid(); table_iter->next()) { @@ -171,7 +181,8 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point) for (const auto & iter : dbs) { const auto & db = iter.second; - if (!isSafeForGC(db, gc_safe_point)) + auto ks_db_id = SchemaNameMapper::getMappedNameKeyspaceID(iter.first); + if (!isSafeForGC(db, gc_safe_point) || ks_db_id != keyspace_id) continue; const auto & db_name = iter.first; diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.h b/dbms/src/TiDB/Schema/SchemaSyncService.h index cb4471ac966..e5e3fa35568 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.h +++ b/dbms/src/TiDB/Schema/SchemaSyncService.h @@ -45,14 +45,14 @@ class SchemaSyncService ~SchemaSyncService(); private: - bool syncSchemas(); + bool syncSchemas(KeyspaceID keyspace_id); struct GCContext { Timestamp last_gc_safe_point = 0; } gc_context; - bool gc(Timestamp gc_safe_point); + bool gc(Timestamp gc_safe_point, KeyspaceID keyspace_id); private: Context & context; diff --git a/dbms/src/TiDB/Schema/SchemaSyncer.h b/dbms/src/TiDB/Schema/SchemaSyncer.h index d0c97ce1a71..4c4d41d978a 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncer.h +++ b/dbms/src/TiDB/Schema/SchemaSyncer.h @@ -40,13 +40,13 @@ class SchemaSyncer /** * Get current version of CH schema. */ - virtual Int64 getCurrentVersion() = 0; + virtual Int64 getCurrentVersion(KeyspaceID keyspace_id) = 0; /** * Synchronize all schemas between TiDB and CH. * @param context */ - virtual bool syncSchemas(Context & context) = 0; + virtual bool syncSchemas(Context & context, KeyspaceID keyspace_id) = 0; virtual void reset() = 0; @@ -54,7 +54,7 @@ class SchemaSyncer virtual TiDB::DBInfoPtr getDBInfoByMappedName(const String & mapped_database_name) = 0; - virtual std::vector fetchAllDBs() = 0; + virtual std::vector fetchAllDBs(KeyspaceID keyspace_id) = 0; }; using SchemaSyncerPtr = std::shared_ptr; diff --git a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h index 4259bdca79b..427bcea366f 100644 --- a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h +++ b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h @@ -33,6 +33,8 @@ namespace ErrorCodes extern const int FAIL_POINT_ERROR; }; +using SchemaVerMap = std::unordered_map; + template struct TiDBSchemaSyncer : public SchemaSyncer { @@ -44,23 +46,22 @@ struct TiDBSchemaSyncer : public SchemaSyncer static constexpr Int64 maxNumberOfDiffs = 100; - Int64 cur_version; + SchemaVerMap cur_versions; std::mutex schema_mutex; - std::unordered_map databases; + KeyspaceDatabaseMap databases; Poco::Logger * log; explicit TiDBSchemaSyncer(KVClusterPtr cluster_) : cluster(std::move(cluster_)) - , cur_version(0) , log(&Poco::Logger::get("SchemaSyncer")) {} bool isTooOldSchema(Int64 cur_ver, Int64 new_version) { return cur_ver == 0 || new_version - cur_ver > maxNumberOfDiffs; } - Getter createSchemaGetter() + Getter createSchemaGetter(KeyspaceID keyspace_id) { [[maybe_unused]] auto tso = cluster->pd_client->getTS(); if constexpr (mock_getter) @@ -69,7 +70,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer } else { - return Getter(cluster.get(), tso); + return Getter(cluster.get(), tso, keyspace_id); } } @@ -82,26 +83,30 @@ struct TiDBSchemaSyncer : public SchemaSyncer std::lock_guard lock(schema_mutex); databases.clear(); - cur_version = 0; + cur_versions.clear(); } - std::vector fetchAllDBs() override + std::vector fetchAllDBs(KeyspaceID keyspace_id) override { - auto getter = createSchemaGetter(); + auto getter = createSchemaGetter(keyspace_id); return getter.listDBs(); } - Int64 getCurrentVersion() override + Int64 getCurrentVersion(KeyspaceID keyspace_id) override { std::lock_guard lock(schema_mutex); - return cur_version; + auto it = cur_versions.find(keyspace_id); + if (it == cur_versions.end()) + return 0; + return it->second; } - bool syncSchemas(Context & context) override + bool syncSchemas(Context & context, KeyspaceID keyspace_id) override { std::lock_guard lock(schema_mutex); + auto cur_version = cur_versions.try_emplace(keyspace_id, 0).first->second; + auto getter = createSchemaGetter(keyspace_id); - auto getter = createSchemaGetter(); Int64 version = getter.getVersion(); if (version <= cur_version) { @@ -110,7 +115,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer Stopwatch watch; SCOPE_EXIT({ GET_METRIC(tiflash_schema_apply_duration_seconds).Observe(watch.elapsedSeconds()); }); - LOG_INFO(log, "Start to sync schemas. current version is: {} and try to sync schema version to: {}", cur_version, version); + LOG_INFO(log, "[keyspace {}] Start to sync schemas. current version is: {} and try to sync schema version to: {}", keyspace_id, cur_version, version); // Show whether the schema mutex is held for a long time or not. GET_METRIC(tiflash_schema_applying).Set(1.0); @@ -125,14 +130,15 @@ struct TiDBSchemaSyncer : public SchemaSyncer // Since TiDB can not make sure the schema diff of the latest schema version X is not empty, under this situation we should set the `cur_version` // to X-1 and try to fetch the schema diff X next time. Int64 version_after_load_diff = 0; - if (version_after_load_diff = tryLoadSchemaDiffs(getter, version, context); version_after_load_diff == -1) + if (version_after_load_diff = tryLoadSchemaDiffs(getter, cur_version, version, context); version_after_load_diff == -1) { GET_METRIC(tiflash_schema_apply_count, type_full).Increment(); version_after_load_diff = loadAllSchema(getter, version, context); } - cur_version = version_after_load_diff; + cur_versions[keyspace_id] = version_after_load_diff; + // TODO: (keyspace) attach keyspace id to the metrics. GET_METRIC(tiflash_schema_version).Set(cur_version); - LOG_INFO(log, "End sync schema, version has been updated to {}{}", cur_version, cur_version == version ? "" : "(latest diff is empty)"); + LOG_INFO(log, "[keyspace {}] End sync schema, version has been updated to {}{}", keyspace_id, cur_version, cur_version == version ? "" : "(latest diff is empty)"); return true; } @@ -161,7 +167,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer // - if latest schema diff is empty, return the (latest_version - 1) // - if schema_diff.regenerate_schema_map == true, need reload all schema info from TiKV, return (-1) // - if error happend, return (-1) - Int64 tryLoadSchemaDiffs(Getter & getter, Int64 latest_version, Context & context) + Int64 tryLoadSchemaDiffs(Getter & getter, Int64 cur_version, Int64 latest_version, Context & context) { if (isTooOldSchema(cur_version, latest_version)) { From a82b8adadaec4926ab5176e9d06d82f5420eadc5 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Tue, 14 Feb 2023 21:27:28 +0800 Subject: [PATCH 02/24] remove IDAsPathUpgrader and format files Signed-off-by: iosmanthus --- .../Coprocessor/DAGStorageInterpreter.cpp | 4 +- dbms/src/Interpreters/IDAsPathUpgrader.cpp | 929 ------------------ .../src/Storages/DeltaMerge/DeltaMergeStore.h | 10 +- dbms/src/Storages/Transaction/TiDB.h | 4 +- 4 files changed, 12 insertions(+), 935 deletions(-) delete mode 100644 dbms/src/Interpreters/IDAsPathUpgrader.cpp diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 19afe6642ea..0819463deb9 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -782,7 +782,7 @@ std::unordered_map DAG { for (auto const physical_table_id : table_scan.getPhysicalTableIDs()) { - auto physical_table_storage = tmt.getStorages().get(keyspace_id,physical_table_id); + auto physical_table_storage = tmt.getStorages().get(keyspace_id, physical_table_id); if (!physical_table_storage) { throw TiFlashException(fmt::format("Table {} doesn't exist.", physical_table_id), Errors::Table::NotExists); @@ -800,7 +800,7 @@ std::unordered_map DAG auto get_and_lock_storage = [&](bool schema_synced, TableID table_id) -> std::tuple { /// Get storage in case it's dropped then re-created. // If schema synced, call getTable without try, leading to exception on table not existing. - auto table_store = tmt.getStorages().get(keyspace_id,table_id); + auto table_store = tmt.getStorages().get(keyspace_id, table_id); if (!table_store) { if (schema_synced) diff --git a/dbms/src/Interpreters/IDAsPathUpgrader.cpp b/dbms/src/Interpreters/IDAsPathUpgrader.cpp deleted file mode 100644 index 3184a1a43a8..00000000000 --- a/dbms/src/Interpreters/IDAsPathUpgrader.cpp +++ /dev/null @@ -1,929 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ -extern const int BAD_ARGUMENTS; -extern const int LOGICAL_ERROR; -extern const int FILE_DOESNT_EXIST; -extern const int SYNTAX_ERROR; -} // namespace ErrorCodes - -static constexpr auto SYSTEM_DATABASE = "system"; - -namespace -{ -std::shared_ptr getDatabaseEngine(const FileProviderPtr & file_provider, const String & filename) -{ - String query; - if (Poco::File(filename).exists()) - { - ReadBufferFromFileProvider in(file_provider, filename, EncryptionPath(filename, ""), 1024); - readStringUntilEOF(query, in); - } - else - { - // only directory exists, "default" database, return "Ordinary" engine by default. - return std::static_pointer_cast(makeASTFunction("Ordinary")); - } - - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "in file " + filename, 0); - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - auto * storage = ast_create_query.storage; - if (storage == nullptr || storage->engine == nullptr || storage->engine->name.empty()) - { - throw Exception("Can not get database engine for file: " + filename, ErrorCodes::LOGICAL_ERROR); - } - - return std::static_pointer_cast(storage->engine->clone()); -} - -// Get from `table_metadata_file` -std::pair getTableInfo(const FileProviderPtr & file_provider, const String & table_metadata_file) -{ - String definition; - if (Poco::File(table_metadata_file).exists()) - { - ReadBufferFromFileProvider in(file_provider, table_metadata_file, EncryptionPath(table_metadata_file, ""), 1024); - readStringUntilEOF(definition, in); - } - else - { - throw Exception("Can not open table schema file: " + table_metadata_file, ErrorCodes::LOGICAL_ERROR); - } - - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, definition.data(), definition.data() + definition.size(), "in file " + table_metadata_file, 0); - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - auto * storage = ast_create_query.storage; - if (storage == nullptr || storage->engine == nullptr || storage->engine->name.empty()) - { - throw Exception("Can not get table engine for file: " + table_metadata_file, ErrorCodes::LOGICAL_ERROR); - } - - TiDB::TableInfo info; - ASTFunction * engine = storage->engine; - const auto * args = typeid_cast(engine->arguments.get()); - if (args == nullptr) - throw Exception("Can not cast table engine arguments", ErrorCodes::BAD_ARGUMENTS); - - const ASTLiteral * table_info_ast = nullptr; - if (engine->name == MutableSupport::delta_tree_storage_name) - { - if (args->children.size() >= 2) - { - table_info_ast = typeid_cast(args->children[1].get()); - } - } - else if (engine->name == MutableSupport::txn_storage_name) - { - if (args->children.size() >= 3) - { - table_info_ast = typeid_cast(args->children[2].get()); - } - } - else - { - throw Exception("Unknown storage engine: " + engine->name, ErrorCodes::LOGICAL_ERROR); - } - - if (table_info_ast && table_info_ast->value.getType() == Field::Types::String) - { - const auto table_info_json = safeGet(table_info_ast->value); - if (!table_info_json.empty()) - { - info.deserialize(table_info_json); - return {ast_create_query.table, info}; - } - } - - throw Exception("Can not get TableInfo for file: " + table_metadata_file, ErrorCodes::BAD_ARGUMENTS); -} - -void renamePath(const String & old_path, const String & new_path, Poco::Logger * log, bool must_success) -{ - if (auto file = Poco::File{old_path}; file.exists()) - { - file.renameTo(new_path); - } - else - { - std::string err_msg = fmt::format(R"(Path "{}" is missing.)", old_path); - if (must_success) - throw Exception(err_msg); - else - LOG_WARNING(log, err_msg); - } -} - -void writeTableDefinitionToFile( - const FileProviderPtr & file_provider, - const String & table_meta_path, - const ASTPtr & query, - bool fsync_metadata) -{ - String table_meta_tmp_path = table_meta_path + ".tmp"; - { - String statement = getTableDefinitionFromCreateQuery(query); - - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. - WriteBufferFromFileProvider out(file_provider, table_meta_tmp_path, EncryptionPath(table_meta_tmp_path, ""), true, nullptr, statement.size(), O_WRONLY | O_CREAT | O_EXCL); - writeString(statement, out); - out.next(); - if (fsync_metadata) - out.sync(); - out.close(); - } - file_provider->renameFile( - table_meta_tmp_path, - EncryptionPath(table_meta_tmp_path, ""), - table_meta_path, - EncryptionPath(table_meta_path, ""), - true); -} - -void writeDatabaseDefinitionToFile( - const FileProviderPtr & file_provider, - const String & database_meta_path, - const ASTPtr & query, - bool fsync_metadata) -{ - String db_meta_tmp_path = database_meta_path + ".tmp"; - { - String statement = getDatabaseDefinitionFromCreateQuery(query); - - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. - WriteBufferFromFileProvider out(file_provider, db_meta_tmp_path, EncryptionPath(db_meta_tmp_path, ""), true, nullptr, statement.size(), O_WRONLY | O_CREAT | O_EXCL); - writeString(statement, out); - out.next(); - if (fsync_metadata) - out.sync(); - out.close(); - } - file_provider->renameFile( - db_meta_tmp_path, - EncryptionPath(db_meta_tmp_path, ""), - database_meta_path, - EncryptionPath(database_meta_path, ""), - true); -} - -ASTPtr parseCreateDatabaseAST(const String & statement) -{ - ParserCreateQuery parser; - const char * pos = statement.data(); - std::string error_msg; - auto ast = tryParseQuery(parser, - pos, - pos + statement.size(), - error_msg, - /*hilite=*/false, - String("in ") + __PRETTY_FUNCTION__, - /*allow_multi_statements=*/false, - 0); - if (!ast) - throw Exception(error_msg, ErrorCodes::SYNTAX_ERROR); - return ast; -} - -// By default, only remove directory if it is empy -void tryRemoveDirectory(const String & directory, Poco::Logger * log, bool recursive = false) -{ - if (auto dir = Poco::File(directory); dir.exists() && dir.isDirectory()) - { - try - { - dir.remove(/*recursive=*/recursive); - } - catch (Poco::DirectoryNotEmptyException &) - { - // just ignore and keep that directory if it is not empty - LOG_WARNING(log, "Can not remove directory: {}, it is not empty", directory); - } - } -} - -// This function will tidy up path and compare if them are the same one. -// For example "/tmp/data/a.sql" is equal to "/tmp//data//a.sql" -inline bool isSamePath(const String & lhs, const String & rhs) -{ - return Poco::Path{lhs}.absolute().toString() == Poco::Path{rhs}.absolute().toString(); -} - -} // namespace - - -// ================================================ -// TableDiskInfo -// ================================================ - -String IDAsPathUpgrader::TableDiskInfo::name() const -{ - // Name in table_info may not be updated, use the name in `ATTACH TABLE ...`. - // The name in table_info will be updated in later schema sync. - return old_name; -} -String IDAsPathUpgrader::TableDiskInfo::newName() const -{ - return mapper->mapTableName(*tidb_table_info); -} -const TiDB::TableInfo & IDAsPathUpgrader::TableDiskInfo::getInfo() const -{ - return *tidb_table_info; -} - -// "metadata/${db_name}/${tbl_name}.sql" -String IDAsPathUpgrader::TableDiskInfo::getMetaFilePath(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getMetaDirectory(root_path) + escapeForFileName(name()) + ".sql"; -} -// "data/${db_name}/${tbl_name}/" -String IDAsPathUpgrader::TableDiskInfo::getDataDirectory( - const String & root_path, - const DatabaseDiskInfo & db, - bool escape_db, - bool escape_tbl) const -{ - String res = db.getDataDirectory(root_path, escape_db); - if (escape_tbl) - res += escapeForFileName(name()); - else - res += name(); - return res + "/"; -} -// "extra_data/${db_name}/${tbl_name}/" -String IDAsPathUpgrader::TableDiskInfo::getExtraDirectory( - const String & root_path, - const DatabaseDiskInfo & db, - bool escape_db, - bool escape_tbl) const -{ - String res = db.getExtraDirectory(root_path, escape_db); - if (escape_tbl) - res += escapeForFileName(name()); - else - res += name(); - return res + "/"; -} - -// "metadata/db_${db_id}/t_${id}.sql" -String IDAsPathUpgrader::TableDiskInfo::getNewMetaFilePath(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getNewMetaDirectory(root_path) + escapeForFileName(newName()) + ".sql"; -} -// "data/t_${id}/" -String IDAsPathUpgrader::TableDiskInfo::getNewDataDirectory(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getNewDataDirectory(root_path) + escapeForFileName(newName()) + "/"; -} -// "extra_data/t_${id}" -String IDAsPathUpgrader::TableDiskInfo::getNewExtraDirectory(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getNewExtraDirectory(root_path) + escapeForFileName(newName()) + "/"; -} - -// ================================================ -// DatabaseDiskInfo -// ================================================ - -void IDAsPathUpgrader::DatabaseDiskInfo::setDBInfo(TiDB::DBInfoPtr info_) -{ - tidb_db_info = info_; -} - -const TiDB::DBInfo & IDAsPathUpgrader::DatabaseDiskInfo::getInfo() const -{ - if (!hasValidTiDBInfo()) - throw Exception("Try to get database info of not inited database: " + name); - return *tidb_db_info; -} - -String IDAsPathUpgrader::DatabaseDiskInfo::newName() const -{ - return mapper->mapDatabaseName(getInfo()); -} - -String IDAsPathUpgrader::DatabaseDiskInfo::getTiDBSerializeInfo() const -{ - if (!hasValidTiDBInfo()) - throw Exception("Try to serialize database info of not inited database: " + name); - return tidb_db_info->serialize(); -} - -// "metadata/${db_name}.sql" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetMetaFilePath(const String & root_path, bool tmp) const -{ - String meta_dir = doGetMetaDirectory(root_path, tmp); - return (endsWith(meta_dir, "/") ? meta_dir.substr(0, meta_dir.size() - 1) : meta_dir) + ".sql"; -} -// "metadata/${db_name}/" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetMetaDirectory(const String & root_path, bool tmp) const -{ - return root_path + (endsWith(root_path, "/") ? "" : "/") + "metadata/" + escapeForFileName(name + (tmp ? TMP_SUFFIX : "")) + "/"; -} -// "data/${db_name}/" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetDataDirectory(const String & root_path, bool escape, bool tmp) const -{ - // Old data path don't do escape for path - if (escape) - return root_path + (endsWith(root_path, "/") ? "" : "/") + "data/" + escapeForFileName(name + (tmp ? TMP_SUFFIX : "")) + "/"; - else - { - // Old extra data path (in PathPool) don't escape for path. - return root_path + (endsWith(root_path, "/") ? "" : "/") + "data/" + name + (tmp ? TMP_SUFFIX : "") + "/"; - } -} -// "extra_data/${db_name}/" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetExtraDirectory(const String & extra_root, bool escape, bool tmp) const -{ - if (escape) - return extra_root + (endsWith(extra_root, "/") ? "" : "/") + escapeForFileName(name + (tmp ? TMP_SUFFIX : "")) + "/"; - else - { - // Old extra data path (in PathPool) don't escape for path. - return extra_root + (endsWith(extra_root, "/") ? "" : "/") + name + (tmp ? TMP_SUFFIX : "") + "/"; - } -} - -// "metadata/db_${id}.sql" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewMetaFilePath(const String & root_path) const -{ - String meta_dir = getNewMetaDirectory(root_path); - return (endsWith(meta_dir, "/") ? meta_dir.substr(0, meta_dir.size() - 1) : meta_dir) + ".sql"; -} -// "metadata/db_${id}/" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewMetaDirectory(const String & root_path) const -{ - return root_path + (endsWith(root_path, "/") ? "" : "/") + "/metadata/" + escapeForFileName(newName()) + "/"; -} -// "data/" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewDataDirectory(const String & root_path) -{ - return root_path + "/data/"; -} -// "extra_data/" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewExtraDirectory(const String & extra_root) -{ - return extra_root + "/"; -} - - -void IDAsPathUpgrader::DatabaseDiskInfo::renameToTmpDirectories(const Context & ctx, Poco::Logger * log) -{ - if (moved_to_tmp) - return; - - auto root_path = ctx.getPath(); - // Rename database meta file if exist - renamePath(doGetMetaFilePath(root_path, false), doGetMetaFilePath(root_path, true), log, false); - // Rename database meta dir - renamePath(doGetMetaDirectory(root_path, false), doGetMetaDirectory(root_path, true), log, true); - - // Rename database data dir - renamePath( // - doGetDataDirectory(root_path, /*escape*/ true, /*tmp*/ false), - doGetDataDirectory(root_path, /*escape*/ true, /*tmp*/ true), - log, - true); - - // Rename database data dir for multi-paths - auto root_pool = ctx.getPathPool(); - for (const auto & extra_path : root_pool.listPaths()) - renamePath( // - doGetExtraDirectory(extra_path, /*escape*/ true, /*tmp*/ false), // - doGetExtraDirectory(extra_path, /*escape*/ true, /*tmp*/ true), - log, - false); - - moved_to_tmp = true; -} - - -// ================================================ -// IDAsPathUpgrader -// ================================================ - -IDAsPathUpgrader::IDAsPathUpgrader(Context & global_ctx_, bool is_mock_, std::unordered_set reserved_databases_) - : global_context(global_ctx_) - , root_path{global_context.getPath()} - , is_mock(is_mock_) - , mapper(is_mock ? std::make_shared() // - : std::make_shared()) - , reserved_databases{std::move(reserved_databases_)} - , log{&Poco::Logger::get("IDAsPathUpgrader")} -{} - -bool IDAsPathUpgrader::needUpgrade() -{ - const auto metadata_path = global_context.getPath() + "/metadata"; - - // For old version, we have database directories and its `.sql` file - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(metadata_path); it != dir_end; ++it) - { - if (!it->isDirectory()) - continue; - - /// For '.svn', '.gitignore' directory and similar. - if (it.name().at(0) == '.') - continue; - - if (it.name() == SYSTEM_DATABASE) - continue; - - String db_name = unescapeForFileName(it.name()); - databases.emplace(db_name, DatabaseDiskInfo{db_name, mapper}); - } - - bool has_old_db_engine = false; - for (auto && [db_name, db_info] : databases) - { - (void)db_name; - const String database_metadata_file = db_info.getMetaFilePath(root_path); - auto engine = getDatabaseEngine(global_context.getFileProvider(), database_metadata_file); - db_info.engine = engine->name; - if (db_info.engine != "TiFlash") - { - has_old_db_engine = true; - LOG_INFO(log, "Find old style of database engine, doing upgrade [path={}] [engine={}]", database_metadata_file, db_info.engine); - } - } - - return has_old_db_engine; -} - -std::vector IDAsPathUpgrader::fetchInfosFromTiDB() const -{ - // Fetch DBs info from TiDB/TiKV - // Note: Not get table info from TiDB, just rename according to TableID in persisted TableInfo - for (size_t i = 0; i < 60; i++) // retry for 3 mins - { - try - { - auto schema_syncer = global_context.getTMTContext().getSchemaSyncer(); - return schema_syncer->fetchAllDBs(NullspaceID); - } - catch (Poco::Exception & e) - { - const int wait_seconds = 3; - LOG_ERROR( - log, - "Upgrade failed because fetch schema error: {}\nWe will sleep for {} seconds and try again.", - e.displayText(), - wait_seconds); - ::sleep(wait_seconds); - } - } - throw Exception("Upgrade failed because fetch schema error."); -} - -static void dropAbsentDatabase( - Context & context, - const String & db_name, - const IDAsPathUpgrader::DatabaseDiskInfo & db_info, - Poco::Logger * log) -{ - if (db_info.hasValidTiDBInfo()) - throw Exception("Invalid call for dropAbsentDatabase for database " + db_name + " with info: " + db_info.getTiDBSerializeInfo()); - - /// tryRemoveDirectory with recursive=true to clean up - - const auto root_path = context.getPath(); - // Remove old metadata dir - const String old_meta_dir = db_info.getMetaDirectory(root_path); - tryRemoveDirectory(old_meta_dir, log, true); - // Remove old metadata file - const String old_meta_file = db_info.getMetaFilePath(root_path); - if (auto file = Poco::File(old_meta_file); file.exists()) - file.remove(); - else - LOG_WARNING(log, "Can not remove database meta file: {}", old_meta_file); - // Remove old data dir - const String old_data_dir = db_info.getDataDirectory(root_path); - tryRemoveDirectory(old_data_dir, log, true); - // not escaped dir created by old PathPool - const String old_data_dir_not_escaped = db_info.getDataDirectory(root_path, false); - tryRemoveDirectory(old_data_dir_not_escaped, log, true); - - const auto & data_extra_paths = context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - tryRemoveDirectory(db_info.getExtraDirectory(extra_root_path), log, true); - tryRemoveDirectory(db_info.getExtraDirectory(extra_root_path, false), log, true); - } -} - -void IDAsPathUpgrader::linkDatabaseTableInfos(const std::vector & all_databases) -{ - for (const auto & db : all_databases) - { - if (auto iter = databases.find(db->name); iter != databases.end()) - { - iter->second.setDBInfo(db); - } - } - - // list all table in old style. - for (auto iter = databases.begin(); iter != databases.end(); /*empty*/) - { - const auto & db_name = iter->first; - auto & db_info = iter->second; - if (!db_info.hasValidTiDBInfo()) - { - // If we can't find it in TiDB, maybe it already dropped. - if (reserved_databases.count(db_name) > 0) - { - // For mock test or develop environment, we may reserve some database - // for convenience. Keep them as what they are. Print warnings and - // ignore it in later upgrade. - LOG_WARNING(log, "Database {} is reserved, ignored in upgrade.", db_name); - } - else - { - // If we keep them as "Ordinary", when user actually create database with - // same name, next time TiFlash restart and will try to do "upgrade" on - // those legacy data, and it will mess everything up. - // Drop them. - dropAbsentDatabase(global_context, db_name, db_info, log); - } - iter = databases.erase(iter); - continue; - } - - if (db_info.engine == "TiFlash") - { - ++iter; - continue; - } - - const String db_meta_dir = db_info.getMetaDirectory(root_path); - std::vector file_names = DatabaseLoading::listSQLFilenames(db_meta_dir, log); - for (const auto & table_filename : file_names) - { - String table_meta_file = db_meta_dir + "/" + table_filename; - // Name in table_info may not be updated, use the name in `ATTACH TABLE ...`. - auto [old_name, table_info] = getTableInfo(global_context.getFileProvider(), table_meta_file); - db_info.tables.emplace_back( // - TableDiskInfo{old_name, std::make_shared(table_info), mapper}); - } - ++iter; - } -} - -void IDAsPathUpgrader::fixNotEscapedDirectories() -{ - for (const auto & [db_name, db_info] : databases) - { - const auto db_name_escaped = escapeForFileName(db_name); - - // database's meta file, meta dir (created by old DatabaseOrdinary) is escaped. - // only need to create data path - if (db_name != db_name_escaped) - { - LOG_INFO(log, "database `{}` fixing name escape to `{}`", db_name, db_name_escaped); - // Create directory for escaped database - auto escaped_db_data_dir = db_info.getDataDirectory(root_path, /*escape=*/true); - if (Poco::File dir(escaped_db_data_dir); !dir.exists()) - dir.createDirectory(); - - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - auto escaped_extra_dir = db_info.getExtraDirectory(extra_root_path, /*escape=*/true); - if (Poco::File dir(escaped_extra_dir); !dir.exists()) - dir.createDirectory(); - } - } - - /// Fix not escaped name for table - for (const auto & table : db_info.tables) - { - const auto table_name_escaped = escapeForFileName(table.name()); - if (db_name_escaped == db_name && table_name_escaped == table.name()) - continue; - - LOG_INFO( - log, - "table `{}`.`{}` fixing name escape to `{}`.`{}`", - db_name, - table.name(), - db_name_escaped, - table_name_escaped); - // Table's metadata don't need to fix. - - // Fix data path. It was create by DatabaseOrdinary and StorageDeltaMerge, - // database name is escaped but table name not. - auto not_escaped_path = table.getDataDirectory(root_path, db_info, /*escape_db*/ true, /*escape_tbl*/ false); - auto escaped_path = table.getDataDirectory(root_path, db_info, /*escape_db*/ true, /*escape_tbl*/ true); - if (auto file = Poco::File{not_escaped_path}; file.exists()) - { - if (auto escaped_dir = Poco::File{escaped_path}; !escaped_dir.exists()) - escaped_dir.createDirectory(); - renamePath(not_escaped_path + "/meta", escaped_path + "/meta", log, true); - renamePath(not_escaped_path + "/data", escaped_path + "/data", log, true); - renamePath(not_escaped_path + "/log", escaped_path + "/log", log, true); - // For the cases that database's name did not need to be escaped but table's name did. - renamePath(not_escaped_path + "/stable", escaped_path + "/stable", log, false); - tryRemoveDirectory(not_escaped_path, log); - } - auto db_tbl_not_escaped_path = not_escaped_path; - if (db_name != db_name_escaped) - { - // For the cases that database's name need to be escaped. - // Stable dir was created by old PathPool, database name and table name were not escaped. - db_tbl_not_escaped_path = table.getDataDirectory(root_path, db_info, false, false); - auto not_escaped_stable = db_tbl_not_escaped_path + "/stable"; - auto escaped_stable = table.getDataDirectory(root_path, db_info, true, true) + "/stable"; - if (auto file = Poco::File{not_escaped_stable}; file.exists()) - renamePath(not_escaped_stable, escaped_stable, log, true); - } - - // Fix extra path. - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - // It was created by old PathPool, both database name and table name are not escaped. - auto not_escaped_extra_path = table.getExtraDirectory(extra_root_path, db_info, /*escape_db*/ false, /*escape_tbl*/ false); - if (isSamePath(not_escaped_extra_path, db_tbl_not_escaped_path)) - continue; - auto escaped_extra_path = table.getExtraDirectory(extra_root_path, db_info, /*escape_db*/ true, /*escape_tbl*/ true); - renamePath(not_escaped_extra_path, escaped_extra_path, log, false); - } - LOG_INFO( - log, - "table `{}`.`{}` fixing name escape to `{}`.`{}` done.", - db_name, - table.name(), - db_name_escaped, - table_name_escaped); - } - - if (db_name != db_name_escaped) - { - // clean not escaped database dir created by old PathPool - const String not_escaped_data_dir = db_info.getDataDirectory(root_path, /*escape*/ false); - tryRemoveDirectory(not_escaped_data_dir, log, true); - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - auto not_escaped_extra_data_dir = db_info.getExtraDirectory(extra_root_path, /*escape*/ false); - if (isSamePath(not_escaped_data_dir, not_escaped_extra_data_dir)) - continue; - tryRemoveDirectory(not_escaped_extra_data_dir, log); - } - } - LOG_INFO(log, "database `{}` fixing name escape to `{}` done.", db_name, db_name_escaped); - } -} - -void IDAsPathUpgrader::resolveConflictDirectories() -{ - std::unordered_set conflict_databases; - for (const auto & [db_name, db_info] : databases) - { - // In theory, user can create database naming "t_xx" and there is cyclic renaming between table and database. - // First detect if there is any database may have cyclic rename with table. - for (const auto & table : db_info.tables) - { - const auto new_tbl_name = table.newName(); - if (auto iter = databases.find(new_tbl_name); iter != databases.end()) - { - conflict_databases.insert(iter->first); - LOG_INFO( - log, - "Detect cyclic renaming between table `{}`.`{}`(new name:{}) and database `{}`", - db_name, - table.name(), - new_tbl_name, - iter->first); - } - } - - // In theory, user can create two database naming "db_xx" and there is cyclic renaming. - // We need to break that cyclic. - const auto new_database_name = db_info.newName(); - if (auto iter = databases.find(new_database_name); iter != databases.end()) - { - conflict_databases.insert(iter->first); - LOG_INFO( - log, - "Detect cyclic renaming between database `{}`(new name:{}) and database `{}`", - db_name, - new_database_name, - iter->first); - } - } - LOG_INFO(log, "Detect {} cyclic renaming", conflict_databases.size()); - for (const auto & db_name : conflict_databases) - { - auto iter = databases.find(db_name); - auto & db_info = iter->second; - LOG_INFO(log, "Move {} to tmp directories..", db_name); - db_info.renameToTmpDirectories(global_context, log); - } -} - -void IDAsPathUpgrader::doRename() -{ - for (const auto & [db_name, db_info] : databases) - { - renameDatabase(db_name, db_info); - } -} - -void IDAsPathUpgrader::renameDatabase(const String & db_name, const DatabaseDiskInfo & db_info) -{ - const auto mapped_db_name = db_info.newName(); - - { - // Create directory for target database - auto new_db_meta_dir = db_info.getNewMetaDirectory(root_path); - Poco::File(new_db_meta_dir).createDirectory(); - } - - // Rename all tables of this database - for (const auto & table : db_info.tables) - { - renameTable(db_name, db_info, mapped_db_name, table); - } - - // Then rename database - LOG_INFO(log, "database `{}` to `{}` renaming", db_name, mapped_db_name); - { - // Recreate metadata file for database - const String new_meta_file = db_info.getNewMetaFilePath(root_path); - const String statement = "ATTACH DATABASE `" + mapped_db_name + "` ENGINE=TiFlash('" + db_info.getTiDBSerializeInfo() + "', 1)\n"; - auto ast = parseCreateDatabaseAST(statement); - const auto & settings = global_context.getSettingsRef(); - writeDatabaseDefinitionToFile(global_context.getFileProvider(), new_meta_file, ast, settings.fsync_metadata); - } - - { - // Remove old metadata dir - const String old_meta_dir = db_info.getMetaDirectory(root_path); - tryRemoveDirectory(old_meta_dir, log); - // Remove old metadata file - const String old_meta_file = db_info.getMetaFilePath(root_path); - if (auto file = Poco::File(old_meta_file); file.exists()) - file.remove(); - else - LOG_WARNING(log, "Can not remove database meta file: {}", old_meta_file); - // Remove old data dir - const String old_data_dir = db_info.getDataDirectory(root_path); - tryRemoveDirectory(old_data_dir, log); - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - tryRemoveDirectory(db_info.getExtraDirectory(extra_root_path), log); - } - } - LOG_INFO(log, "database `{}` to `{}` rename done.", db_name, mapped_db_name); -} - -void IDAsPathUpgrader::renameTable( - const String & db_name, - const DatabaseDiskInfo & db_info, - const String & mapped_db_name, - const TableDiskInfo & table) -{ - const auto mapped_table_name = table.newName(); - LOG_INFO( - log, - "table `{}`.`{}` to `{}`.`{}` renaming", - db_name, - table.name(), - mapped_db_name, - mapped_table_name); - - String old_tbl_data_path; - { - // Former data path use ${path}/data/${database}/${table}/ as data path. - // Rename it to ${path}/data/${mapped_table_name}. - old_tbl_data_path = table.getDataDirectory(root_path, db_info); - renamePath(old_tbl_data_path, table.getNewDataDirectory(root_path, db_info), log, true); - } - - { - // Rename data path for multi disk - auto data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - auto old_tbl_extra_data_path = table.getExtraDirectory(extra_root_path, db_info); - if (isSamePath(old_tbl_extra_data_path, old_tbl_data_path)) - continue; - renamePath(old_tbl_extra_data_path, table.getNewExtraDirectory(extra_root_path, db_info), log, false); - } - } - - // Recreate metadata file - { - auto old_tbl_meta_file = table.getMetaFilePath(root_path, db_info); - auto ast = DatabaseLoading::getQueryFromMetadata(global_context, old_tbl_meta_file, /*throw_on_error=*/true); - if (!ast) - throw Exception("There is no metadata file for table " + table.name() + ", expected file: " + old_tbl_meta_file, - ErrorCodes::FILE_DOESNT_EXIST); - - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - ast_create_query.table = mapped_table_name; - ASTStorage * storage_ast = ast_create_query.storage; - TiDB::TableInfo table_info = table.getInfo(); // get a copy - if (table_info.is_partition_table) - { - LOG_INFO( - log, - "partition table `{}`.`{}` to `{}`.`{}` update table info", - db_name, - table.name(), - mapped_db_name, - mapped_table_name); - // Old partition name is "${table_name}_${physical_id}" while new name is "t_${physical_id}" - // If it is a partition table, we need to update TiDB::TableInfo::name - do - { - if (!storage_ast || !storage_ast->engine) - break; - auto * args = typeid_cast(storage_ast->engine->arguments.get()); - if (!args) - break; - - table_info.name = mapper->mapPartitionName(table_info); - std::shared_ptr literal = std::make_shared(Field(table_info.serialize())); - if (args->children.size() == 1) - args->children.emplace_back(literal); - else if (args->children.size() >= 2) - args->children.at(1) = literal; - } while (false); - } - - const String new_tbl_meta_file = table.getNewMetaFilePath(root_path, db_info); - const auto & settings = global_context.getSettingsRef(); - writeTableDefinitionToFile(global_context.getFileProvider(), new_tbl_meta_file, ast, settings.fsync_metadata); - - // Remove old metadata file - if (auto file = Poco::File(old_tbl_meta_file); file.exists()) - file.remove(); - } - - LOG_INFO( - log, - "table `{}`.`{}` to `{}`.`{}` rename done.", - db_name, - table.name(), - mapped_db_name, - mapped_table_name); -} - -void IDAsPathUpgrader::doUpgrade() -{ - auto all_databases = fetchInfosFromTiDB(); - linkDatabaseTableInfos(all_databases); - fixNotEscapedDirectories(); - // Check if destination db / tbl file exists and resolve conflict - resolveConflictDirectories(); - // Rename - doRename(); -} - -} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index d544ab76c0e..638817459a4 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -405,8 +405,14 @@ class DeltaMergeStore : private boost::noncopyable StoreStats getStoreStats(); SegmentsStats getSegmentsStats(); - bool isCommonHandle() const { return is_common_handle; } - size_t getRowKeyColumnSize() const { return rowkey_column_size; } + bool isCommonHandle() const + { + return is_common_handle; + } + size_t getRowKeyColumnSize() const + { + return rowkey_column_size; + } public: /// Methods mainly used by region split. diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index 18ae9c32527..5f818bf10dd 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -47,11 +47,11 @@ namespace TiDB { using DB::ColumnID; using DB::DatabaseID; +using DB::KeyspaceID; +using DB::NullspaceID; using DB::String; using DB::TableID; using DB::Timestamp; -using DB::KeyspaceID; -using DB::NullspaceID; // Column types. // In format: From cb87b2111ea833cece1f2f90a007ca06a18d1134 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Tue, 14 Feb 2023 22:31:23 +0800 Subject: [PATCH 03/24] add license header Signed-off-by: iosmanthus --- dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp | 14 ++++++++++++++ dbms/src/Storages/Transaction/KeyspaceSnapshot.h | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp b/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp index 3ab93dd1ed9..ad6841f1f9e 100644 --- a/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp +++ b/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp @@ -1,3 +1,17 @@ +// Copyright 2023 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include diff --git a/dbms/src/Storages/Transaction/KeyspaceSnapshot.h b/dbms/src/Storages/Transaction/KeyspaceSnapshot.h index d56dbd675e3..c7580a8e177 100644 --- a/dbms/src/Storages/Transaction/KeyspaceSnapshot.h +++ b/dbms/src/Storages/Transaction/KeyspaceSnapshot.h @@ -1,3 +1,17 @@ +// Copyright 2023 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #pragma GCC diagnostic push From bdf46293c3efcdad2c14618ed19d4a3b76e4aecb Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 16 Feb 2023 11:34:49 +0800 Subject: [PATCH 04/24] better exception check --- dbms/src/Storages/Transaction/RegionTable.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionTable.cpp b/dbms/src/Storages/Transaction/RegionTable.cpp index 289e67bf83b..53c3aceb432 100644 --- a/dbms/src/Storages/Transaction/RegionTable.cpp +++ b/dbms/src/Storages/Transaction/RegionTable.cpp @@ -25,6 +25,8 @@ #include #include +#include "Common/Exception.h" + namespace DB { namespace ErrorCodes @@ -452,17 +454,18 @@ void RegionTable::extendRegionRange(const RegionID region_id, const RegionRangeK auto keyspace_id = region_range_keys.getKeyspaceID(); auto table_id = region_range_keys.getMappedTableID(); - auto ks_tb_id = KeyspaceTableID{keyspace_id, table_id}; + auto ks_tbl_id = KeyspaceTableID{keyspace_id, table_id}; auto new_handle_range = region_range_keys.rawKeys(); if (auto it = regions.find(region_id); it != regions.end()) { - if (ks_tb_id != it->second) - throw Exception(std::string(__PRETTY_FUNCTION__) + ": table id " + std::to_string(table_id) + " not match previous one " - + std::to_string(it->second.second) + " in regions " + std::to_string(region_id), - ErrorCodes::LOGICAL_ERROR); + RUNTIME_CHECK_MSG( + ks_tbl_id == it->second, + "{}: table id not match the previous one" + ", region_id={} keyspace_id={} table_id={}, old_keyspace_id={} old_table_id={}", + __PRETTY_FUNCTION__, region_id, keyspace_id, table_id, it->second.first, it->second.second); - InternalRegion & internal_region = doGetInternalRegion(ks_tb_id, region_id); + InternalRegion & internal_region = doGetInternalRegion(ks_tbl_id, region_id); if (*(internal_region.range_in_table.first) <= *(new_handle_range.first) && *(internal_region.range_in_table.second) >= *(new_handle_range.second)) { From 43b67c2e47ee8cdf6aafb3274f222008bde91837 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 16 Feb 2023 11:42:27 +0800 Subject: [PATCH 05/24] Format files --- dbms/src/Storages/Transaction/RegionTable.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/RegionTable.cpp b/dbms/src/Storages/Transaction/RegionTable.cpp index 53c3aceb432..9a791cebf7d 100644 --- a/dbms/src/Storages/Transaction/RegionTable.cpp +++ b/dbms/src/Storages/Transaction/RegionTable.cpp @@ -463,7 +463,12 @@ void RegionTable::extendRegionRange(const RegionID region_id, const RegionRangeK ks_tbl_id == it->second, "{}: table id not match the previous one" ", region_id={} keyspace_id={} table_id={}, old_keyspace_id={} old_table_id={}", - __PRETTY_FUNCTION__, region_id, keyspace_id, table_id, it->second.first, it->second.second); + __PRETTY_FUNCTION__, + region_id, + keyspace_id, + table_id, + it->second.first, + it->second.second); InternalRegion & internal_region = doGetInternalRegion(ks_tbl_id, region_id); if (*(internal_region.range_in_table.first) <= *(new_handle_range.first) From 25a58439994d009958f6366bddead115fe929c6e Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 16 Feb 2023 11:55:53 +0800 Subject: [PATCH 06/24] Fix compile error --- dbms/src/Common/tests/gtest_redact.cpp | 4 ++-- .../Storages/Page/V3/Universal/UniversalPageStorageService.h | 1 + dbms/src/Storages/Transaction/RegionTable.cpp | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Common/tests/gtest_redact.cpp b/dbms/src/Common/tests/gtest_redact.cpp index 68407b1c2b4..70fe330f4c8 100644 --- a/dbms/src/Common/tests/gtest_redact.cpp +++ b/dbms/src/Common/tests/gtest_redact.cpp @@ -19,12 +19,12 @@ namespace DB { namespace tests { -TEST(RedactLog_test, Basic) +TEST(RedactLogTest, Basic) { const char * test_key = "\x01\x0a\xff"; const size_t key_sz = strlen(test_key); - const DB::HandleID test_handle = 10009; + const /*DB::HandleID*/ Int64 test_handle = 10009; Redact::setRedactLog(false); EXPECT_EQ(Redact::keyToDebugString(test_key, key_sz), "010AFF"); diff --git a/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h b/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h index 99e3d573365..b69165d0f38 100644 --- a/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h +++ b/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h @@ -16,6 +16,7 @@ #include #include +#include namespace DB { diff --git a/dbms/src/Storages/Transaction/RegionTable.cpp b/dbms/src/Storages/Transaction/RegionTable.cpp index 9a791cebf7d..c5e7d1127dd 100644 --- a/dbms/src/Storages/Transaction/RegionTable.cpp +++ b/dbms/src/Storages/Transaction/RegionTable.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -25,8 +26,6 @@ #include #include -#include "Common/Exception.h" - namespace DB { namespace ErrorCodes From 264e1dbd3e2ea84cd7aaaf0d90f68696b4aab650 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Mon, 20 Feb 2023 21:00:46 +0800 Subject: [PATCH 07/24] attach keyspace_id for tiflash cop/mpp requests Signed-off-by: iosmanthus --- contrib/client-c | 2 +- contrib/kvproto | 2 +- dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp | 2 +- dbms/src/Storages/GCManager.cpp | 4 +++- dbms/src/Storages/StorageDisaggregated.cpp | 1 + 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/contrib/client-c b/contrib/client-c index b4ab3efa024..b2a6ce6e678 160000 --- a/contrib/client-c +++ b/contrib/client-c @@ -1 +1 @@ -Subproject commit b4ab3efa024386ad028876907bdbab682cc2f2b3 +Subproject commit b2a6ce6e6786cfbbc9fa26942b66eb043b2f7960 diff --git a/contrib/kvproto b/contrib/kvproto index 579d6ab1bff..c6df78cc9de 160000 --- a/contrib/kvproto +++ b/contrib/kvproto @@ -1 +1 @@ -Subproject commit 579d6ab1bfffd5e5ca71436dff59bcb5843188c4 +Subproject commit c6df78cc9dea61bdf913ec8bac49e245980c1a94 diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 0819463deb9..34541b4a645 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -445,7 +445,7 @@ std::vector DAGStorageInterpreter::buildCopTasks( std::multimap meta_data; meta_data.emplace("is_remote_read", "true"); - auto tasks = pingcap::coprocessor::buildCopTasks(bo, cluster, remote_request.key_ranges, req, store_type, &Poco::Logger::get("pingcap/coprocessor"), std::move(meta_data), [&] { + auto tasks = pingcap::coprocessor::buildCopTasks(bo, cluster, remote_request.key_ranges, req, store_type, dagContext().getKeyspaceID(), &Poco::Logger::get("pingcap/coprocessor"), std::move(meta_data), [&] { GET_METRIC(tiflash_coprocessor_request_count, type_remote_read_sent).Increment(); }); all_tasks.insert(all_tasks.end(), tasks.begin(), tasks.end()); diff --git a/dbms/src/Storages/GCManager.cpp b/dbms/src/Storages/GCManager.cpp index eea5db3d22a..dc17308ff97 100644 --- a/dbms/src/Storages/GCManager.cpp +++ b/dbms/src/Storages/GCManager.cpp @@ -97,7 +97,9 @@ bool GCManager::work() } if (iter == storages.end()) iter = storages.begin(); - next_keyspace_table_id = iter->first; + + if (iter != storages.end()) + next_keyspace_table_id = iter->first; LOG_DEBUG(log, "End GC and next gc will start with keyspace {}, table id: {}", next_keyspace_table_id.first, next_keyspace_table_id.second); gc_check_stop_watch.restart(); // Always return false diff --git a/dbms/src/Storages/StorageDisaggregated.cpp b/dbms/src/Storages/StorageDisaggregated.cpp index 0f4ac2bec7f..2dcd3b3139b 100644 --- a/dbms/src/Storages/StorageDisaggregated.cpp +++ b/dbms/src/Storages/StorageDisaggregated.cpp @@ -116,6 +116,7 @@ StorageDisaggregated::RequestAndRegionIDs StorageDisaggregated::buildDispatchMPP { auto dispatch_req = std::make_shared<::mpp::DispatchTaskRequest>(); ::mpp::TaskMeta * dispatch_req_meta = dispatch_req->mutable_meta(); + dispatch_req_meta->set_keyspace_id(context.getDAGContext()->getKeyspaceID()); dispatch_req_meta->set_start_ts(sender_target_mpp_task_id.query_id.start_ts); dispatch_req_meta->set_query_ts(sender_target_mpp_task_id.query_id.query_ts); dispatch_req_meta->set_local_query_id(sender_target_mpp_task_id.query_id.local_query_id); From 62a18bb6a2609f59ed0562c4e6572d1bc7946f57 Mon Sep 17 00:00:00 2001 From: yongman Date: Tue, 21 Feb 2023 16:55:17 +0800 Subject: [PATCH 08/24] Sync keyspace schema in exclusive task Signed-off-by: yongman --- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 102 ++++++++++++++------- dbms/src/TiDB/Schema/SchemaSyncService.h | 5 + 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index d498bd4c85f..3a0f908a36b 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -37,54 +37,92 @@ SchemaSyncService::SchemaSyncService(DB::Context & context_) , background_pool(context_.getBackgroundPool()) , log(&Poco::Logger::get("SchemaSyncService")) { + // Add task for adding and removing keyspace sync schema tasks. handle = background_pool.addTask( [&, this] { - String stage; auto keyspaces = context.getTMTContext().getStorages().getAllKeyspaces(); - KeyspaceID cur_ks = NullspaceID; - bool done_anything = false; - try + std::shared_lock lock(ks_map_mutex); + + // Add new sync schema task for new keyspace. + for (auto const iter: keyspaces) { - auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient()); - for (auto const iter : keyspaces) + auto ks = iter.first; + if (!ks_handle_map.count(ks)) { - auto ks = iter.first; - LOG_DEBUG(log, "Auto sync schema for keyspace {}", ks); - cur_ks = ks; - /// Do sync schema first, then gc. - /// They must be performed synchronously, - /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. - // GC safe point must be obtained ahead of syncing schema. - stage = "Sync schemas"; - done_anything = syncSchemas(ks); - if (done_anything) - GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); - - stage = "GC"; - done_anything = gc(gc_safe_point, ks); + LOG_INFO(log, "add sync schema task for keyspace {}", ks); + + auto task_handle = background_pool.addTask( + [&, this, ks] { + String stage; + bool done_anything = false; + try + { + LOG_DEBUG(log, "sync schema for keyspace {}", ks); + /// Do sync schema first, then gc. + /// They must be performed synchronously, + /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. + // GC safe point must be obtained ahead of syncing schema. + auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient()); + stage = "Sync schemas"; + done_anything = syncSchemas(ks); + if (done_anything) + GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); + + stage = "GC"; + done_anything = gc(gc_safe_point, ks); + + return done_anything; + } + catch (const Exception & e) + { + LOG_ERROR(log, "[keyspace {}] {} failed by {} \n stack : {}", ks, stage, e.displayText(), e.getStackTrace().toString()); + } + catch (const Poco::Exception & e) + { + LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.displayText()); + } + catch (const std::exception & e) + { + LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.what()); + } + return false; + }, + false + ); + + ks_handle_map.emplace(ks, task_handle); } - return done_anything; - } - catch (const Exception & e) - { - LOG_ERROR(log, "[keyspace {}] {} failed by {} \n stack : {}", cur_ks, stage, e.displayText(), e.getStackTrace().toString()); - } - catch (const Poco::Exception & e) - { - LOG_ERROR(log, "[keyspace {}] {} failed by {}", cur_ks, stage, e.displayText()); } - catch (const std::exception & e) + + // Remove stale sync schema task. + for (auto const & iter: ks_handle_map) { - LOG_ERROR(log, "[keyspace {}] {} failed by {}", cur_ks, stage, e.what()); + auto ks = iter.first; + auto task_handle = iter.second; + + if (!keyspaces.count(ks)) + { + LOG_INFO(log, "remove sync schema task for keyspace {}", ks); + ks_handle_map.erase(ks); + + background_pool.removeTask(task_handle); + } } + return false; }, - false); + false + ); } SchemaSyncService::~SchemaSyncService() { background_pool.removeTask(handle); + for (auto const & iter: ks_handle_map) + { + auto task_handle = iter.second; + background_pool.removeTask(task_handle); + } } bool SchemaSyncService::syncSchemas(KeyspaceID keyspace_id) diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.h b/dbms/src/TiDB/Schema/SchemaSyncService.h index e5e3fa35568..ae2ebf285ce 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.h +++ b/dbms/src/TiDB/Schema/SchemaSyncService.h @@ -19,6 +19,8 @@ #include #include +#include +#include namespace Poco { @@ -62,6 +64,9 @@ class SchemaSyncService BackgroundProcessingPool & background_pool; BackgroundProcessingPool::TaskHandle handle; + mutable std::shared_mutex ks_map_mutex; + std::unordered_map ks_handle_map; + Poco::Logger * log; }; From 7b9c63de8ae8760319f85f9bfb9e9240368f3f63 Mon Sep 17 00:00:00 2001 From: yongman Date: Tue, 21 Feb 2023 17:16:56 +0800 Subject: [PATCH 09/24] Fix lock type in schema sync Signed-off-by: yongman --- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index 3a0f908a36b..fff8110c9a4 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -41,7 +41,7 @@ SchemaSyncService::SchemaSyncService(DB::Context & context_) handle = background_pool.addTask( [&, this] { auto keyspaces = context.getTMTContext().getStorages().getAllKeyspaces(); - std::shared_lock lock(ks_map_mutex); + std::unique_lock lock(ks_map_mutex); // Add new sync schema task for new keyspace. for (auto const iter: keyspaces) From 47400e1c1c455b1f0d54eceec613881c38b98e11 Mon Sep 17 00:00:00 2001 From: yongman Date: Tue, 28 Feb 2023 18:27:30 +0800 Subject: [PATCH 10/24] Split lambda to functions Signed-off-by: yongman --- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 134 +++++++++++---------- dbms/src/TiDB/Schema/SchemaSyncService.h | 4 + 2 files changed, 74 insertions(+), 64 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index fff8110c9a4..9b20b3a4eba 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -40,79 +40,85 @@ SchemaSyncService::SchemaSyncService(DB::Context & context_) // Add task for adding and removing keyspace sync schema tasks. handle = background_pool.addTask( [&, this] { - auto keyspaces = context.getTMTContext().getStorages().getAllKeyspaces(); - std::unique_lock lock(ks_map_mutex); + addKeyspaceGCTasks(); + removeKeyspaceGCTasks(); - // Add new sync schema task for new keyspace. - for (auto const iter: keyspaces) - { - auto ks = iter.first; - if (!ks_handle_map.count(ks)) - { - LOG_INFO(log, "add sync schema task for keyspace {}", ks); + return false; + }, + false); +} - auto task_handle = background_pool.addTask( - [&, this, ks] { - String stage; - bool done_anything = false; - try - { - LOG_DEBUG(log, "sync schema for keyspace {}", ks); - /// Do sync schema first, then gc. - /// They must be performed synchronously, - /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. - // GC safe point must be obtained ahead of syncing schema. - auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient()); - stage = "Sync schemas"; - done_anything = syncSchemas(ks); - if (done_anything) - GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); +void SchemaSyncService::addKeyspaceGCTasks() +{ + auto keyspaces = context.getTMTContext().getStorages().getAllKeyspaces(); + std::unique_lock lock(ks_map_mutex); - stage = "GC"; - done_anything = gc(gc_safe_point, ks); + // Add new sync schema task for new keyspace. + for (auto const iter : keyspaces) + { + auto ks = iter.first; + if (!ks_handle_map.count(ks)) + { + LOG_INFO(log, "add sync schema task for keyspace {}", ks); - return done_anything; - } - catch (const Exception & e) - { - LOG_ERROR(log, "[keyspace {}] {} failed by {} \n stack : {}", ks, stage, e.displayText(), e.getStackTrace().toString()); - } - catch (const Poco::Exception & e) - { - LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.displayText()); - } - catch (const std::exception & e) - { - LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.what()); - } - return false; - }, - false - ); + auto task_handle = background_pool.addTask( + [&, this, ks] { + String stage; + bool done_anything = false; + try + { + LOG_DEBUG(log, "sync schema for keyspace {}", ks); + /// Do sync schema first, then gc. + /// They must be performed synchronously, + /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. + // GC safe point must be obtained ahead of syncing schema. + auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient()); + stage = "Sync schemas"; + done_anything = syncSchemas(ks); + if (done_anything) + GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); - ks_handle_map.emplace(ks, task_handle); - } - } + stage = "GC"; + done_anything = gc(gc_safe_point, ks); - // Remove stale sync schema task. - for (auto const & iter: ks_handle_map) - { - auto ks = iter.first; - auto task_handle = iter.second; + return done_anything; + } + catch (const Exception & e) + { + LOG_ERROR(log, "[keyspace {}] {} failed by {} \n stack : {}", ks, stage, e.displayText(), e.getStackTrace().toString()); + } + catch (const Poco::Exception & e) + { + LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.displayText()); + } + catch (const std::exception & e) + { + LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.what()); + } + return false; + }, + false); - if (!keyspaces.count(ks)) - { - LOG_INFO(log, "remove sync schema task for keyspace {}", ks); - ks_handle_map.erase(ks); + ks_handle_map.emplace(ks, task_handle); + } + } +} - background_pool.removeTask(task_handle); - } - } +void SchemaSyncService::removeKeyspaceGCTasks() +{ + auto keyspaces = context.getTMTContext().getStorages().getAllKeyspaces(); + std::unique_lock lock(ks_map_mutex); - return false; - }, - false - ); + // Remove stale sync schema task. + for (auto const & [ks, task_handle] : ks_handle_map) + { + if (!keyspaces.count(ks)) + { + LOG_INFO(log, "remove sync schema task for keyspace {}", ks); + ks_handle_map.erase(ks); + background_pool.removeTask(task_handle); + } + } } SchemaSyncService::~SchemaSyncService() diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.h b/dbms/src/TiDB/Schema/SchemaSyncService.h index ae2ebf285ce..3a00c6fdbd0 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.h +++ b/dbms/src/TiDB/Schema/SchemaSyncService.h @@ -56,6 +56,9 @@ class SchemaSyncService bool gc(Timestamp gc_safe_point, KeyspaceID keyspace_id); + void addKeyspaceGCTasks(); + void removeKeyspaceGCTasks(); + private: Context & context; @@ -65,6 +68,7 @@ class SchemaSyncService BackgroundProcessingPool::TaskHandle handle; mutable std::shared_mutex ks_map_mutex; + // Handles for each keyspace schema sync task. std::unordered_map ks_handle_map; Poco::Logger * log; From 990a7ec59a2743d3036e5b03ef12d7903813ee4b Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Tue, 7 Mar 2023 17:57:52 +0800 Subject: [PATCH 11/24] format Signed-off-by: iosmanthus --- dbms/src/Storages/Transaction/Types.h | 2 +- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/Types.h b/dbms/src/Storages/Transaction/Types.h index 14804cd37c1..e35e01fc09e 100644 --- a/dbms/src/Storages/Transaction/Types.h +++ b/dbms/src/Storages/Transaction/Types.h @@ -15,8 +15,8 @@ #pragma once #include -#include #include +#include #include #include diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index e4270cf9d8b..e59fb4d7445 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -124,7 +124,7 @@ void SchemaSyncService::removeKeyspaceGCTasks() SchemaSyncService::~SchemaSyncService() { background_pool.removeTask(handle); - for (auto const & iter: ks_handle_map) + for (auto const & iter : ks_handle_map) { auto task_handle = iter.second; background_pool.removeTask(task_handle); From 32c2e2f1a3ffd0e5cb3d1e2943765b46b13ff47e Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Tue, 7 Mar 2023 22:20:20 +0800 Subject: [PATCH 12/24] manual compact request support keyspace Signed-off-by: iosmanthus --- contrib/kvproto | 2 +- dbms/src/Flash/Management/ManualCompact.cpp | 10 ++++++---- dbms/src/Flash/Management/ManualCompact.h | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/contrib/kvproto b/contrib/kvproto index bfdb1d7eb15..daba7cb42f7 160000 --- a/contrib/kvproto +++ b/contrib/kvproto @@ -1 +1 @@ -Subproject commit bfdb1d7eb15712cff7b6d833f757662da3a4f7f4 +Subproject commit daba7cb42f7edc54d62f4c9e685fe1a0e91ad375 diff --git a/dbms/src/Flash/Management/ManualCompact.cpp b/dbms/src/Flash/Management/ManualCompact.cpp index 95ba868e6b6..44b72f204c8 100644 --- a/dbms/src/Flash/Management/ManualCompact.cpp +++ b/dbms/src/Flash/Management/ManualCompact.cpp @@ -38,11 +38,12 @@ ManualCompactManager::ManualCompactManager(const Context & global_context_, cons grpc::Status ManualCompactManager::handleRequest(const ::kvrpcpb::CompactRequest * request, ::kvrpcpb::CompactResponse * response) { + auto ks_tbl_id = KeyspaceTableID{request->keyspace_id(), request->logical_table_id()}; { std::lock_guard lock(mutex); // Check whether there are duplicated executions. - if (unsync_active_logical_table_ids.count(request->logical_table_id())) + if (unsync_active_logical_table_ids.count(ks_tbl_id)) { response->mutable_error()->mutable_err_compact_in_progress(); response->set_has_remaining(false); @@ -57,12 +58,12 @@ grpc::Status ManualCompactManager::handleRequest(const ::kvrpcpb::CompactRequest return grpc::Status::OK; } - unsync_active_logical_table_ids.insert(request->logical_table_id()); + unsync_active_logical_table_ids.insert(ks_tbl_id); unsync_running_or_pending_tasks++; } SCOPE_EXIT({ std::lock_guard lock(mutex); - unsync_active_logical_table_ids.erase(request->logical_table_id()); + unsync_active_logical_table_ids.erase(ks_tbl_id); unsync_running_or_pending_tasks--; }); @@ -101,7 +102,7 @@ grpc::Status ManualCompactManager::doWork(const ::kvrpcpb::CompactRequest * requ { const auto & tmt_context = global_context.getTMTContext(); // TODO(iosmanthus): support compact keyspace tables; - auto storage = tmt_context.getStorages().get(NullspaceID, request->physical_table_id()); + auto storage = tmt_context.getStorages().get(request->keyspace_id(), request->physical_table_id()); if (storage == nullptr) { response->mutable_error()->mutable_err_physical_table_not_exist(); @@ -163,6 +164,7 @@ grpc::Status ManualCompactManager::doWork(const ::kvrpcpb::CompactRequest * requ Stopwatch timer; + // TODO(iosmanthus): attach keyspace id for this logger. LOG_INFO(log, "Manual compaction begin for table {}, start_key = {}", request->physical_table_id(), start_key.toDebugString()); // Repeatedly merge multiple segments as much as possible. diff --git a/dbms/src/Flash/Management/ManualCompact.h b/dbms/src/Flash/Management/ManualCompact.h index 68e8d77cfc6..2786deb5c0c 100644 --- a/dbms/src/Flash/Management/ManualCompact.h +++ b/dbms/src/Flash/Management/ManualCompact.h @@ -15,6 +15,7 @@ #include #include +#include #include #include #pragma GCC diagnostic push @@ -78,7 +79,7 @@ class ManualCompactManager : private boost::noncopyable /// When there is a task containing the same logical_table running, /// the task will be rejected. - std::set unsync_active_logical_table_ids = {}; + std::unordered_set> unsync_active_logical_table_ids = {}; size_t unsync_running_or_pending_tasks = 0; From afc4912dffb6b15ab733cbd0738a4063e2ca5475 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Wed, 8 Mar 2023 20:56:17 +0800 Subject: [PATCH 13/24] address comments from @JaySon-Huang 1. refine some log messages, add a child logger with keyspace id info 2. add some comments for ManagedStorages's interfaces Signed-off-by: iosmanthus --- dbms/src/Storages/GCManager.cpp | 10 ++++++---- dbms/src/Storages/Transaction/TMTStorages.h | 3 +++ dbms/src/TiDB/Schema/SchemaBuilder.h | 2 +- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 17 +++++++++-------- dbms/src/TiDB/Schema/TiDBSchemaSyncer.h | 5 +++-- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/dbms/src/Storages/GCManager.cpp b/dbms/src/Storages/GCManager.cpp index 20918556d7e..8ee4b7b0129 100644 --- a/dbms/src/Storages/GCManager.cpp +++ b/dbms/src/Storages/GCManager.cpp @@ -48,8 +48,8 @@ bool GCManager::work() // Get a storage snapshot with weak_ptrs first // TODO: avoid gc on storage which have no data? std::map> storages; - for (const auto & [ks_tb_id, storage] : global_context.getTMTContext().getStorages().getAllStorage()) - storages.emplace(ks_tb_id, storage); + for (const auto & [ks_tbl_id, storage] : global_context.getTMTContext().getStorages().getAllStorage()) + storages.emplace(ks_tbl_id, storage); auto iter = storages.begin(); if (next_keyspace_table_id != KeyspaceTableID{NullspaceID, InvalidTableID}) iter = storages.lower_bound(next_keyspace_table_id); @@ -74,13 +74,15 @@ bool GCManager::work() try { + auto keyspace_id = storage->getTableInfo().keyspace_id; + auto ks_log = log->getChild(fmt::format("keyspace={}", keyspace_id)); TableLockHolder table_read_lock = storage->lockForShare(RWLock::NO_QUERY); // Block this thread and do GC on the storage - // It is OK if any schema changes is apply to the storage while doing GC, so we + // It is OK if any schema changes is applied to the storage while doing GC, so we // do not acquire structure lock on the storage. auto gc_segments_num = storage->onSyncGc(gc_segments_limit, DM::GCOptions::newAll()); gc_segments_limit = gc_segments_limit - gc_segments_num; - LOG_TRACE(log, "GCManager gc {} segments of keyspace {}, table {}", gc_segments_num, storage->getTableInfo().keyspace_id, storage->getTableInfo().id); + LOG_TRACE(ks_log, "GCManager gc {} segments of table {}", gc_segments_num, storage->getTableInfo().id); // Reach the limit on the number of segments to be gc, stop here if (gc_segments_limit <= 0) break; diff --git a/dbms/src/Storages/Transaction/TMTStorages.h b/dbms/src/Storages/Transaction/TMTStorages.h index c5be3f8f777..070c1778aa9 100644 --- a/dbms/src/Storages/Transaction/TMTStorages.h +++ b/dbms/src/Storages/Transaction/TMTStorages.h @@ -34,8 +34,11 @@ class ManagedStorages : private boost::noncopyable public: void put(ManageableStoragePtr storage); + // Get storage by keyspace and table id ManageableStoragePtr get(KeyspaceID keyspace_id, TableID table_id) const; + // Get all the storages of all the keyspaces in this instance. StorageMap getAllStorage() const; + // Get all the existed keyspaces in this instance. KeyspaceSet getAllKeyspaces() const; ManageableStoragePtr getByName(const std::string & db, const std::string & table, bool include_tombstone) const; diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.h b/dbms/src/TiDB/Schema/SchemaBuilder.h index 55888af43ed..88b715c29ac 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.h +++ b/dbms/src/TiDB/Schema/SchemaBuilder.h @@ -34,7 +34,7 @@ struct SchemaBuilder Int64 target_version; - KeyspaceID keyspace_id; + const KeyspaceID keyspace_id; LoggerPtr log; diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index e59fb4d7445..50868f96b56 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -59,15 +59,15 @@ void SchemaSyncService::addKeyspaceGCTasks() auto ks = iter.first; if (!ks_handle_map.count(ks)) { - LOG_INFO(log, "add sync schema task for keyspace {}", ks); - + auto ks_log = log->getChild(fmt::format("keyspace={}", ks)); + LOG_INFO(ks_log, "add sync schema task"); auto task_handle = background_pool.addTask( - [&, this, ks] { + [&, this, ks, ks_log] { String stage; bool done_anything = false; try { - LOG_DEBUG(log, "sync schema for keyspace {}", ks); + LOG_DEBUG(ks_log, "auto sync schema", ks); /// Do sync schema first, then gc. /// They must be performed synchronously, /// otherwise table may get mis-GC-ed if RECOVER was not properly synced caused by schema sync pause but GC runs too aggressively. @@ -85,15 +85,15 @@ void SchemaSyncService::addKeyspaceGCTasks() } catch (const Exception & e) { - LOG_ERROR(log, "[keyspace {}] {} failed by {} \n stack : {}", ks, stage, e.displayText(), e.getStackTrace().toString()); + LOG_ERROR(ks_log, "{} failed by {} \n stack : {}", stage, e.displayText(), e.getStackTrace().toString()); } catch (const Poco::Exception & e) { - LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.displayText()); + LOG_ERROR(ks_log, "{} failed by {}", stage, e.displayText()); } catch (const std::exception & e) { - LOG_ERROR(log, "[keyspace {}] {} failed by {}", ks, stage, e.what()); + LOG_ERROR(ks_log, "{} failed by {}", stage, e.what()); } return false; }, @@ -114,7 +114,8 @@ void SchemaSyncService::removeKeyspaceGCTasks() { if (!keyspaces.count(ks)) { - LOG_INFO(log, "remove sync schema task for keyspace {}", ks); + auto ks_log = log->getChild(fmt::format("keyspace={}", ks)); + LOG_INFO(ks_log, "remove sync schema task"); ks_handle_map.erase(ks); background_pool.removeTask(task_handle); } diff --git a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h index 862815ebdf8..d180cb46bf8 100644 --- a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h +++ b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h @@ -105,6 +105,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer bool syncSchemas(Context & context, KeyspaceID keyspace_id) override { std::lock_guard lock(schema_mutex); + auto ks_log = log->getChild(fmt::format("keyspace={}", keyspace_id)); auto cur_version = cur_versions.try_emplace(keyspace_id, 0).first->second; auto getter = createSchemaGetter(keyspace_id); @@ -116,7 +117,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer Stopwatch watch; SCOPE_EXIT({ GET_METRIC(tiflash_schema_apply_duration_seconds).Observe(watch.elapsedSeconds()); }); - LOG_INFO(log, "[keyspace {}] Start to sync schemas. current version is: {} and try to sync schema version to: {}", keyspace_id, cur_version, version); + LOG_INFO(ks_log, "Start to sync schemas. current version is: {} and try to sync schema version to: {}", keyspace_id, cur_version, version); // Show whether the schema mutex is held for a long time or not. GET_METRIC(tiflash_schema_applying).Set(1.0); @@ -139,7 +140,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer cur_versions[keyspace_id] = version_after_load_diff; // TODO: (keyspace) attach keyspace id to the metrics. GET_METRIC(tiflash_schema_version).Set(cur_version); - LOG_INFO(log, "[keyspace {}] End sync schema, version has been updated to {}{}", keyspace_id, cur_version, cur_version == version ? "" : "(latest diff is empty)"); + LOG_INFO(ks_log, "End sync schema, version has been updated to {}{}", keyspace_id, cur_version, cur_version == version ? "" : "(latest diff is empty)"); return true; } From fc73d3c5e03bd9534163560e5b2dfc7de887b7ff Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Wed, 8 Mar 2023 21:32:20 +0800 Subject: [PATCH 14/24] add default value for table_id and keyspace_id in RegionRangeKeys Signed-off-by: iosmanthus --- dbms/src/Storages/Transaction/RegionRangeKeys.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionRangeKeys.h b/dbms/src/Storages/Transaction/RegionRangeKeys.h index a69bdb6ccb3..7f695ed8114 100644 --- a/dbms/src/Storages/Transaction/RegionRangeKeys.h +++ b/dbms/src/Storages/Transaction/RegionRangeKeys.h @@ -61,8 +61,8 @@ class RegionRangeKeys : boost::noncopyable private: RegionRange ori; std::pair raw; - TableID mapped_table_id; - KeyspaceID keyspace_id; + TableID mapped_table_id = InvalidTableID; + KeyspaceID keyspace_id = NullspaceID; }; } // namespace DB From bd475c7378cc741908e293a8069353dd3e6ca0df Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 9 Mar 2023 13:32:16 +0800 Subject: [PATCH 15/24] Small log fixes Signed-off-by: JaySon-Huang --- dbms/src/TiDB/Schema/SchemaBuilder.h | 2 +- dbms/src/TiDB/Schema/TiDBSchemaSyncer.h | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.h b/dbms/src/TiDB/Schema/SchemaBuilder.h index 88b715c29ac..e45e495810e 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.h +++ b/dbms/src/TiDB/Schema/SchemaBuilder.h @@ -44,7 +44,7 @@ struct SchemaBuilder , databases(dbs_) , target_version(version) , keyspace_id(getter_.getKeyspaceID()) - , log(Logger::get()) + , log(Logger::get(fmt::format("keyspace={}", keyspace_id))) {} void applyDiff(const SchemaDiff & diff); diff --git a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h index d180cb46bf8..f33beaa9361 100644 --- a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h +++ b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h @@ -117,7 +117,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer Stopwatch watch; SCOPE_EXIT({ GET_METRIC(tiflash_schema_apply_duration_seconds).Observe(watch.elapsedSeconds()); }); - LOG_INFO(ks_log, "Start to sync schemas. current version is: {} and try to sync schema version to: {}", keyspace_id, cur_version, version); + LOG_INFO(ks_log, "Start to sync schemas. current version is: {} and try to sync schema version to: {}", cur_version, version); // Show whether the schema mutex is held for a long time or not. GET_METRIC(tiflash_schema_applying).Set(1.0); @@ -132,7 +132,7 @@ struct TiDBSchemaSyncer : public SchemaSyncer // Since TiDB can not make sure the schema diff of the latest schema version X is not empty, under this situation we should set the `cur_version` // to X-1 and try to fetch the schema diff X next time. Int64 version_after_load_diff = 0; - if (version_after_load_diff = tryLoadSchemaDiffs(getter, cur_version, version, context); version_after_load_diff == -1) + if (version_after_load_diff = tryLoadSchemaDiffs(getter, cur_version, version, context, ks_log); version_after_load_diff == -1) { GET_METRIC(tiflash_schema_apply_count, type_full).Increment(); version_after_load_diff = loadAllSchema(getter, version, context); @@ -168,15 +168,15 @@ struct TiDBSchemaSyncer : public SchemaSyncer // - if latest schema diff is not empty, return the (latest_version) // - if latest schema diff is empty, return the (latest_version - 1) // - if schema_diff.regenerate_schema_map == true, need reload all schema info from TiKV, return (-1) - // - if error happend, return (-1) - Int64 tryLoadSchemaDiffs(Getter & getter, Int64 cur_version, Int64 latest_version, Context & context) + // - if error happens, return (-1) + Int64 tryLoadSchemaDiffs(Getter & getter, Int64 cur_version, Int64 latest_version, Context & context, const LoggerPtr & ks_log) { if (isTooOldSchema(cur_version, latest_version)) { return -1; } - LOG_DEBUG(log, "Try load schema diffs."); + LOG_DEBUG(ks_log, "Try load schema diffs."); Int64 used_version = cur_version; // First get all schema diff from `cur_version` to `latest_version`. Only apply the schema diff(s) if we fetch all @@ -187,12 +187,12 @@ struct TiDBSchemaSyncer : public SchemaSyncer used_version++; diffs.push_back(getter.getSchemaDiff(used_version)); } - LOG_DEBUG(log, "End load schema diffs with total {} entries.", diffs.size()); + LOG_DEBUG(ks_log, "End load schema diffs with total {} entries.", diffs.size()); if (diffs.empty()) { - LOG_WARNING(log, "Schema Diff is empty."); + LOG_WARNING(ks_log, "Schema Diff is empty."); return -1; } // Since the latest schema diff may be empty, and schemaBuilder may need to update the latest version for storageDeltaMerge, @@ -249,19 +249,19 @@ struct TiDBSchemaSyncer : public SchemaSyncer throw; } GET_METRIC(tiflash_schema_apply_count, type_failed).Increment(); - LOG_WARNING(log, "apply diff meets exception : {} \n stack is {}", e.displayText(), e.getStackTrace().toString()); + LOG_WARNING(ks_log, "apply diff meets exception : {} \n stack is {}", e.displayText(), e.getStackTrace().toString()); return -1; } catch (Poco::Exception & e) { GET_METRIC(tiflash_schema_apply_count, type_failed).Increment(); - LOG_WARNING(log, "apply diff meets exception : {}", e.displayText()); + LOG_WARNING(ks_log, "apply diff meets exception : {}", e.displayText()); return -1; } catch (std::exception & e) { GET_METRIC(tiflash_schema_apply_count, type_failed).Increment(); - LOG_WARNING(log, "apply diff meets exception : {}", e.what()); + LOG_WARNING(ks_log, "apply diff meets exception : {}", e.what()); return -1; } From 6e2efa510eb2246d43a54126ea94046d2d7d31d9 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 9 Mar 2023 15:32:33 +0800 Subject: [PATCH 16/24] Small log fixes Signed-off-by: JaySon-Huang --- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index 50868f96b56..a85075812da 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -149,7 +149,9 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) if (gc_safe_point == gc_context.last_gc_safe_point) return false; - LOG_INFO(log, "Performing GC using safe point {}", gc_safe_point); + auto ks_log = log->getChild(fmt::format("keyspace={}", keyspace_id)); + + LOG_INFO(ks_log, "Performing GC using safe point {}", gc_safe_point); // The storages that are ready for gc std::vector> storages_to_gc; @@ -195,7 +197,7 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) return db_info ? SchemaNameMapper().debugCanonicalName(*db_info, table_info) : "(" + database_name + ")." + SchemaNameMapper().debugTableName(table_info); }(); - LOG_INFO(log, "Physically dropping table {}", canonical_name); + LOG_INFO(ks_log, "Physically dropping table {}", canonical_name); auto drop_query = std::make_shared(); drop_query->database = std::move(database_name); drop_query->table = std::move(table_name); @@ -206,7 +208,7 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) { InterpreterDropQuery drop_interpreter(ast_drop_query, context); drop_interpreter.execute(); - LOG_INFO(log, "Physically dropped table {}", canonical_name); + LOG_INFO(ks_log, "Physically dropped table {}", canonical_name); } catch (DB::Exception & e) { @@ -217,7 +219,7 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) err_msg = "locking attempt has timed out!"; // ignore verbose stack for this error else err_msg = getCurrentExceptionMessage(true); - LOG_INFO(log, "Physically drop table {} is skipped, reason: {}", canonical_name, err_msg); + LOG_INFO(ks_log, "Physically drop table {} is skipped, reason: {}", canonical_name, err_msg); } } storages_to_gc.clear(); @@ -238,11 +240,11 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) { // There should be something wrong, maybe a read lock of a table is held for a long time. // Just ignore and try to collect this database next time. - LOG_INFO(log, "Physically drop database {} is skipped, reason: {} tables left", db_name, num_tables); + LOG_INFO(ks_log, "Physically drop database {} is skipped, reason: {} tables left", db_name, num_tables); continue; } - LOG_INFO(log, "Physically dropping database {}", db_name); + LOG_INFO(ks_log, "Physically dropping database {}", db_name); auto drop_query = std::make_shared(); drop_query->database = db_name; drop_query->if_exists = true; @@ -252,7 +254,7 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) { InterpreterDropQuery drop_interpreter(ast_drop_query, context); drop_interpreter.execute(); - LOG_INFO(log, "Physically dropped database {}", db_name); + LOG_INFO(ks_log, "Physically dropped database {}", db_name); } catch (DB::Exception & e) { @@ -262,19 +264,19 @@ bool SchemaSyncService::gc(Timestamp gc_safe_point, KeyspaceID keyspace_id) err_msg = "locking attempt has timed out!"; // ignore verbose stack for this error else err_msg = getCurrentExceptionMessage(true); - LOG_INFO(log, "Physically drop database {} is skipped, reason: {}", db_name, err_msg); + LOG_INFO(ks_log, "Physically drop database {} is skipped, reason: {}", db_name, err_msg); } } if (succeeded) { gc_context.last_gc_safe_point = gc_safe_point; - LOG_INFO(log, "Performed GC using safe point {}", gc_safe_point); + LOG_INFO(ks_log, "Performed GC using safe point {}", gc_safe_point); } else { // Don't update last_gc_safe_point and retry later - LOG_INFO(log, "Performed GC using safe point {} meet error, will try again later", gc_safe_point); + LOG_INFO(ks_log, "Performed GC using safe point {} meet error, will try again later", gc_safe_point); } return true; From e5c527d0f88c0efbd1cf27fb997b8d028e53b5ac Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 9 Mar 2023 16:51:22 +0800 Subject: [PATCH 17/24] preserve old schema sync api Signed-off-by: iosmanthus --- .../Transaction/ProxyFFIStatusService.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp b/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp index cb8179a535d..85b7f5c078c 100644 --- a/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp +++ b/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp @@ -38,21 +38,31 @@ HttpRequestRes HandleHttpRequestSyncStatus( auto * log = &Poco::Logger::get("HandleHttpRequestSyncStatus"); LOG_DEBUG(log, "handling sync status request, path: {}, api_name: {}", path, api_name); - // Query schema: /keyspace/{keyspace_id}/table/{table_id} + // Try to handle sync status request with old schema. + // Old schema: /{table_id} + // New schema: /keyspace/{keyspace_id}/table/{table_id} auto query = path.substr(api_name.size()); std::vector query_parts; boost::split(query_parts, query, boost::is_any_of("/")); - if (query_parts.size() < 4 || query_parts[0] != "keyspace" || query_parts[2] != "table") + if (query_parts.size() != 1 && (query_parts.size() != 4 || query_parts[0] != "keyspace" || query_parts[2] != "table")) { LOG_ERROR(log, "invalid SyncStatus request: {}", query); status = HttpRequestStatus::ErrorParam; return HttpRequestRes{.status = status, .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{}}}; } + try { - keyspace_id = std::stoll(query_parts[1]); - table_id = std::stoll(query_parts[3]); + if (query_parts.size() == 4) + { + keyspace_id = std::stoll(query_parts[1]); + table_id = std::stoll(query_parts[3]); + } + else + { + table_id = std::stoll(query_parts[0]); + } } catch (...) { From a9f53be03e1aa52b89cf7e3484ab1dd9b8be962f Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 9 Mar 2023 17:46:34 +0800 Subject: [PATCH 18/24] address comments from @JaySon-Huang 1. move api_version for TiFlashStorageConfig 2. refine comments for TMTStorages Signed-off-by: iosmanthus --- dbms/src/Server/Server.cpp | 9 +++++---- dbms/src/Server/StorageConfigParser.cpp | 5 +++++ dbms/src/Server/StorageConfigParser.h | 1 + dbms/src/Storages/Transaction/TMTStorages.h | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index a564227e795..7dfbce9268a 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -873,7 +873,6 @@ int Server::main(const std::vector & /*args*/) TiFlashProxyConfig proxy_conf(config()); - auto api_version = config().getInt("api_version", 1); EngineStoreServerWrap tiflash_instance_wrap{}; auto helper = GetEngineStoreServerHelper( @@ -1000,6 +999,8 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Using format_version={} (default settings).", STORAGE_FORMAT_CURRENT.identifier); } + LOG_INFO(log, "Using api_version={}", storage_config.api_version); + global_context->initializePathCapacityMetric( // global_capacity_quota, // storage_config.main_data_paths, @@ -1181,7 +1182,7 @@ int Server::main(const std::vector & /*args*/) if (updated) { auto raft_config = TiFlashRaftConfig::parseSettings(*config, log); - auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, api_version, log); + auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, storage_config.api_version, log); global_context->getTMTContext().updateSecurityConfig(std::move(raft_config), std::move(cluster_config)); LOG_DEBUG(log, "TMTContext updated security config"); } @@ -1238,7 +1239,7 @@ int Server::main(const std::vector & /*args*/) { /// create TMTContext - auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, api_version, log); + auto cluster_config = getClusterConfig(global_context->getSecurityConfig(), raft_config, storage_config.api_version, log); global_context->createTMTContext(raft_config, std::move(cluster_config)); global_context->getTMTContext().reloadConfig(config()); } @@ -1261,7 +1262,7 @@ int Server::main(const std::vector & /*args*/) { /// Then, sync schemas with TiDB, and initialize schema sync service. /// If in API V2 mode, each keyspace's schema is fetch lazily. - if (api_version == 1) + if (storage_config.api_version == 1) { for (int i = 0; i < 60; i++) // retry for 3 mins { diff --git a/dbms/src/Server/StorageConfigParser.cpp b/dbms/src/Server/StorageConfigParser.cpp index 8e4945dfc18..743bd3dfd3c 100644 --- a/dbms/src/Server/StorageConfigParser.cpp +++ b/dbms/src/Server/StorageConfigParser.cpp @@ -210,6 +210,11 @@ void TiFlashStorageConfig::parseMisc(const String & storage_section, const Logge format_version = *version; } + if (auto version = table->get_qualified_as("api_version"); version) + { + api_version = *version; + } + auto get_bool_config_or_default = [&](const String & name, bool default_value) { if (auto value = table->get_qualified_as(name); value) { diff --git a/dbms/src/Server/StorageConfigParser.h b/dbms/src/Server/StorageConfigParser.h index 70740891f58..0f94bde6bcf 100644 --- a/dbms/src/Server/StorageConfigParser.h +++ b/dbms/src/Server/StorageConfigParser.h @@ -137,6 +137,7 @@ struct TiFlashStorageConfig UInt64 format_version = 0; bool lazily_init_store = true; + UInt64 api_version = 1; StorageS3Config s3_config; StorageRemoteCacheConfig remote_cache_config; diff --git a/dbms/src/Storages/Transaction/TMTStorages.h b/dbms/src/Storages/Transaction/TMTStorages.h index 070c1778aa9..55c897c715b 100644 --- a/dbms/src/Storages/Transaction/TMTStorages.h +++ b/dbms/src/Storages/Transaction/TMTStorages.h @@ -38,7 +38,7 @@ class ManagedStorages : private boost::noncopyable ManageableStoragePtr get(KeyspaceID keyspace_id, TableID table_id) const; // Get all the storages of all the keyspaces in this instance. StorageMap getAllStorage() const; - // Get all the existed keyspaces in this instance. + // Get all the existing keyspaces in this instance. A map of `{KeySpaceID => num of physical tables}`. KeyspaceSet getAllKeyspaces() const; ManageableStoragePtr getByName(const std::string & db, const std::string & table, bool include_tombstone) const; From d9a51db024604923d6d8794bf4e91b414a5f9f03 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 9 Mar 2023 18:09:36 +0800 Subject: [PATCH 19/24] use std::string_view instead of char[4] for KEYSPACE_PREFIX Signed-off-by: iosmanthus --- dbms/src/TiDB/Schema/SchemaNameMapper.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaNameMapper.h b/dbms/src/TiDB/Schema/SchemaNameMapper.h index 7fdebbb6f01..515a28d9fc4 100644 --- a/dbms/src/TiDB/Schema/SchemaNameMapper.h +++ b/dbms/src/TiDB/Schema/SchemaNameMapper.h @@ -26,12 +26,12 @@ struct SchemaNameMapper static constexpr auto DATABASE_PREFIX = "db_"; static constexpr auto TABLE_PREFIX = "t_"; - static constexpr auto KEYSPACE_PREFIX = "ks_"; + static constexpr std::string_view KEYSPACE_PREFIX = "ks_"; static KeyspaceID getMappedNameKeyspaceID(const String & name) { - auto keyspace_prefix_len = std::strlen(KEYSPACE_PREFIX); + auto keyspace_prefix_len = KEYSPACE_PREFIX.length(); auto pos = name.find(KEYSPACE_PREFIX); if (pos == String::npos) return NullspaceID; @@ -43,7 +43,7 @@ struct SchemaNameMapper static String map2Keyspace(KeyspaceID keyspace_id, const String & name) { - return keyspace_id == NullspaceID ? name : KEYSPACE_PREFIX + std::to_string(keyspace_id) + "_" + name; + return keyspace_id == NullspaceID ? name : KEYSPACE_PREFIX.data() + std::to_string(keyspace_id) + "_" + name; } virtual String mapDatabaseName(const TiDB::DBInfo & db_info) const From 96f4445f0e16580b19d5e78d4dd7e5c7b59c0cf4 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 9 Mar 2023 18:14:02 +0800 Subject: [PATCH 20/24] refine comments Signed-off-by: iosmanthus --- dbms/src/Flash/Management/ManualCompact.cpp | 1 - dbms/src/Storages/StorageDisaggregated.cpp | 1 + dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Flash/Management/ManualCompact.cpp b/dbms/src/Flash/Management/ManualCompact.cpp index 44b72f204c8..006e53ebd8e 100644 --- a/dbms/src/Flash/Management/ManualCompact.cpp +++ b/dbms/src/Flash/Management/ManualCompact.cpp @@ -101,7 +101,6 @@ grpc::Status ManualCompactManager::doWorkWithCatch(const ::kvrpcpb::CompactReque grpc::Status ManualCompactManager::doWork(const ::kvrpcpb::CompactRequest * request, ::kvrpcpb::CompactResponse * response) { const auto & tmt_context = global_context.getTMTContext(); - // TODO(iosmanthus): support compact keyspace tables; auto storage = tmt_context.getStorages().get(request->keyspace_id(), request->physical_table_id()); if (storage == nullptr) { diff --git a/dbms/src/Storages/StorageDisaggregated.cpp b/dbms/src/Storages/StorageDisaggregated.cpp index c5985d10a16..d3ce53f6cc9 100644 --- a/dbms/src/Storages/StorageDisaggregated.cpp +++ b/dbms/src/Storages/StorageDisaggregated.cpp @@ -117,6 +117,7 @@ StorageDisaggregated::RequestAndRegionIDs StorageDisaggregated::buildDispatchMPP { auto dispatch_req = std::make_shared<::mpp::DispatchTaskRequest>(); ::mpp::TaskMeta * dispatch_req_meta = dispatch_req->mutable_meta(); + // TODO(iosmanthus): support S3 remote read in keyspace mode. dispatch_req_meta->set_keyspace_id(context.getDAGContext()->getKeyspaceID()); dispatch_req_meta->set_start_ts(sender_target_mpp_task_id.query_id.start_ts); dispatch_req_meta->set_query_ts(sender_target_mpp_task_id.query_id.query_ts); diff --git a/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp b/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp index ad6841f1f9e..4e83ea9a60c 100644 --- a/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp +++ b/dbms/src/Storages/Transaction/KeyspaceSnapshot.cpp @@ -43,7 +43,7 @@ std::string KeyspaceSnapshot::Get(pingcap::kv::Backoffer & bo, const std::string KeyspaceScanner KeyspaceSnapshot::Scan(const std::string & begin, const std::string & end) { auto inner = snap.Scan(encodeKey(begin), encodeKey(end)); - return KeyspaceScanner(inner, !prefix.empty()); + return KeyspaceScanner(inner, /* need_cut_ */ !prefix.empty()); } std::string KeyspaceSnapshot::encodeKey(const std::string & key) From 57ccc1e13232952226609a221f94e9d70ead6881 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 9 Mar 2023 20:54:23 +0800 Subject: [PATCH 21/24] bump client-c to latest commit of api-v2-for-release-6.6 Signed-off-by: iosmanthus --- contrib/client-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/client-c b/contrib/client-c index 21ac6a27a70..dbfaee9ec43 160000 --- a/contrib/client-c +++ b/contrib/client-c @@ -1 +1 @@ -Subproject commit 21ac6a27a70fad4a6da80ffd29b2f6f6a91f9937 +Subproject commit dbfaee9ec43e0c65707326a1ede94fd065f9baa7 From 08925c4c9281b4de400cad155e2d4184328a4e37 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Fri, 10 Mar 2023 15:54:55 +0800 Subject: [PATCH 22/24] update client-c and kvproto's commit and add api_version docs in example config Signed-off-by: iosmanthus --- contrib/client-c | 2 +- contrib/kvproto | 2 +- etc/config-template.toml | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/client-c b/contrib/client-c index dbfaee9ec43..ddeed5fc44a 160000 --- a/contrib/client-c +++ b/contrib/client-c @@ -1 +1 @@ -Subproject commit dbfaee9ec43e0c65707326a1ede94fd065f9baa7 +Subproject commit ddeed5fc44a85ddf81efc9eaa601087f0a6ee6fd diff --git a/contrib/kvproto b/contrib/kvproto index daba7cb42f7..145f0534a0e 160000 --- a/contrib/kvproto +++ b/contrib/kvproto @@ -1 +1 @@ -Subproject commit daba7cb42f7edc54d62f4c9e685fe1a0e91ad375 +Subproject commit 145f0534a0eb7e953c78563c4b50e3877c120409 diff --git a/etc/config-template.toml b/etc/config-template.toml index d46bb632e0c..82cdfab1ba0 100644 --- a/etc/config-template.toml +++ b/etc/config-template.toml @@ -30,6 +30,10 @@ ## The storage format version in storage engine. Valid values: 1, 2. ## format_version = 2 +## The storage api-version for the TiFlash engine. Valid values: 1, 2, default is 1. +## while using api-version = 2, multiple keyspace data could be stored in a single storage. +## api_version = 1 + ## If there are multiple SSD disks on the machine, ## specify the path list on `storage.main.dir` can improve TiFlash performance. From bdb71caf7089e580a9243dcbcc94209fd5b295fb Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Fri, 10 Mar 2023 17:24:04 +0800 Subject: [PATCH 23/24] update client-c to master Signed-off-by: iosmanthus --- contrib/client-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/client-c b/contrib/client-c index ddeed5fc44a..04d408143e5 160000 --- a/contrib/client-c +++ b/contrib/client-c @@ -1 +1 @@ -Subproject commit ddeed5fc44a85ddf81efc9eaa601087f0a6ee6fd +Subproject commit 04d408143e5ceb01799ec642560c39dc1a0a373f From 9c193ac249610d9622d63fc8552c16232e5c03c5 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Fri, 10 Mar 2023 17:31:36 +0800 Subject: [PATCH 24/24] git checkout master contrib/tiflash-proxy Signed-off-by: iosmanthus --- contrib/tiflash-proxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy index 6793d377dac..c9070bf6270 160000 --- a/contrib/tiflash-proxy +++ b/contrib/tiflash-proxy @@ -1 +1 @@ -Subproject commit 6793d377dacdfe921c4a6cb428fe6c6bd0dc1a0e +Subproject commit c9070bf62704407092bdff406719708ebf9d8962