From 320330c96c0694268cf20db264dd9ce4834ae3bc Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Fri, 24 Dec 2021 10:16:02 +0800 Subject: [PATCH] Support meta upgrade v3 --- src/clients/meta/MetaClient.h | 2 +- src/daemons/CMakeLists.txt | 1 + src/daemons/MetaDaemonInit.cpp | 32 +- src/interface/meta.thrift | 9 - src/kvstore/CMakeLists.txt | 4 - src/meta/CMakeLists.txt | 2 + src/meta/MetaServiceHandler.cpp | 1 - src/meta/MetaVersionMan.cpp | 107 +- src/meta/MetaVersionMan.h | 11 +- .../admin/ListClusterInfoProcessor.cpp | 4 +- .../processors/zone/MergeZoneProcessor.cpp | 1 - .../processors/zone/SplitZoneProcessor.cpp | 186 --- src/meta/processors/zone/SplitZoneProcessor.h | 34 - src/meta/test/MetaClientTest.cpp | 1002 ----------------- src/meta/test/ProcessorTest.cpp | 1 - src/meta/upgrade/CMakeLists.txt | 3 + src/meta/upgrade/MetaDataUpgrade.cpp | 136 ++- src/meta/upgrade/MetaDataUpgrade.h | 32 +- src/meta/upgrade/v2/CMakeLists.txt | 24 + src/meta/upgrade/v2/MetaServiceUtilsV2.cpp | 32 + src/meta/upgrade/v2/MetaServiceUtilsV2.h | 33 + src/meta/upgrade/v2/meta.thrift | 81 ++ src/parser/parser.yy | 24 +- src/parser/test/ParserTest.cpp | 41 +- src/storage/test/CMakeLists.txt | 1 + src/tools/db-dump/CMakeLists.txt | 1 + src/tools/db-upgrade/CMakeLists.txt | 1 + src/tools/meta-dump/CMakeLists.txt | 1 + src/tools/simple-kv-verify/CMakeLists.txt | 1 + src/tools/storage-perf/CMakeLists.txt | 1 + 30 files changed, 491 insertions(+), 1318 deletions(-) delete mode 100644 src/meta/processors/zone/SplitZoneProcessor.cpp delete mode 100644 src/meta/processors/zone/SplitZoneProcessor.h create mode 100644 src/meta/upgrade/v2/CMakeLists.txt create mode 100644 src/meta/upgrade/v2/MetaServiceUtilsV2.cpp create mode 100644 src/meta/upgrade/v2/MetaServiceUtilsV2.h create mode 100644 src/meta/upgrade/v2/meta.thrift diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 38ae54353ff..314fc6df8aa 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -762,7 +762,7 @@ class MetaClient { std::atomic metadLastUpdateTime_{0}; int64_t metaServerVersion_{-1}; - static constexpr int64_t EXPECT_META_VERSION = 2; + static constexpr int64_t EXPECT_META_VERSION = 3; // leadersLock_ is used to protect leadersInfo folly::RWSpinLock leadersLock_; diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 8d8d68e680f..532c5ba9700 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -98,6 +98,7 @@ nebula_add_executable( $ $ $ + $ $ ${common_deps} ${storage_meta_deps} diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index c41b1e02c06..09640d5dc58 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -138,26 +138,22 @@ std::unique_ptr initKV(std::vector p LOG(ERROR) << "Meta version is invalid"; return nullptr; } else if (version == nebula::meta::MetaVersion::V1) { - if (leader == localhost) { - LOG(INFO) << "I am leader, begin upgrade meta data"; - // need to upgrade the v1.0 meta data format to v2.0 meta data format - auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); - if (!ret.ok()) { - LOG(ERROR) << ret; - return nullptr; - } - } else { - LOG(INFO) << "I am follower, wait for leader to sync upgrade"; - while (version != nebula::meta::MetaVersion::V2) { - VLOG(1) << "Waiting for leader to upgrade"; - sleep(1); - version = nebula::meta::MetaVersionMan::getMetaVersionFromKV(kvstore.get()); - } + auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; + } + + nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V2); + } else if (version == nebula::meta::MetaVersion::V2) { + LOG(INFO) << "version 3"; + auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; } - } - if (leader == localhost) { - nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get()); + nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V3); } LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId; diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index 23d2ad93d30..1ee7bbacb68 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -877,18 +877,9 @@ struct DropZoneReq { 1: binary zone_name, } -<<<<<<< HEAD struct DivideZoneReq { 1: binary zone_name, 2: map> (cpp.template = "std::unordered_map") zone_items, -======= -struct SplitZoneReq { - 1: binary zone_name, - 2: binary one_zone_name, - 3: list one_zone_hosts, - 4: binary another_zone_name, - 5: list another_zone_hosts, ->>>>>>> support zone operations } struct RenameZoneReq { diff --git a/src/kvstore/CMakeLists.txt b/src/kvstore/CMakeLists.txt index 261fbd5d15e..549be980087 100644 --- a/src/kvstore/CMakeLists.txt +++ b/src/kvstore/CMakeLists.txt @@ -19,12 +19,8 @@ nebula_add_library( nebula_add_subdirectory(raftex) nebula_add_subdirectory(wal) -<<<<<<< HEAD nebula_add_subdirectory(stats) nebula_add_subdirectory(test) -======= -#nebula_add_subdirectory(test) ->>>>>>> support zone operations #nebula_add_subdirectory(plugins) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 54873ae9c70..fa1e8f6e09d 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -113,12 +113,14 @@ nebula_add_library( add_dependencies( meta_version_man_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) set(meta_test_deps $ $ $ + $ $ $ $ diff --git a/src/meta/MetaServiceHandler.cpp b/src/meta/MetaServiceHandler.cpp index ebe09e22eb0..997d38ed894 100644 --- a/src/meta/MetaServiceHandler.cpp +++ b/src/meta/MetaServiceHandler.cpp @@ -72,7 +72,6 @@ #include "meta/processors/zone/ListZonesProcessor.h" #include "meta/processors/zone/MergeZoneProcessor.h" #include "meta/processors/zone/RenameZoneProcessor.h" -#include "meta/processors/zone/SplitZoneProcessor.h" #define RETURN_FUTURE(processor) \ auto f = processor->getFuture(); \ diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index dd61af74c2e..9bd77a4043d 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -5,10 +5,12 @@ #include "meta/MetaVersionMan.h" +#include "meta/ActiveHostsMan.h" #include "meta/processors/job/JobDescription.h" #include "meta/processors/job/JobUtils.h" #include "meta/upgrade/MetaDataUpgrade.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DEFINE_bool(null_type, true, "set schema to support null type"); DEFINE_bool(print_info, false, "enable to print the rewrite data"); @@ -49,12 +51,11 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { } // static -bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { +bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version) { CHECK_NOTNULL(kv); - auto v2 = MetaVersion::V2; std::vector data; data.emplace_back(kMetaVersionKey, - std::string(reinterpret_cast(&v2), sizeof(MetaVersion))); + std::string(reinterpret_cast(&version), sizeof(MetaVersion))); bool ret = true; folly::Baton baton; kv->asyncMultiPut( @@ -63,7 +64,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { LOG(ERROR) << "Put failed, error: " << static_cast(code); ret = false; } else { - LOG(INFO) << "Write meta version 2 succeeds"; + LOG(INFO) << "Write meta version 3 succeeds"; } baton.post(); }); @@ -80,7 +81,28 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { LOG(ERROR) << "Create snapshot failed: " << snapshot; return Status::Error("Create snapshot failed"); } - auto status = doUpgrade(kv); + auto status = doUpgradeV1ToV2(kv); + if (!status.ok()) { + // rollback by snapshot + return status; + } + // delete snapshot file + auto dmRet = kv->dropCheckpoint(kDefaultSpaceId, snapshot); + if (dmRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually"; + } + return Status::OK(); +} + +Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVStore* kv) { + CHECK_NOTNULL(kv); + auto snapshot = folly::format("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str(); + auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); + if (meteRet.isLeftType()) { + LOG(ERROR) << "Create snapshot failed: " << snapshot; + return Status::Error("Create snapshot failed"); + } + auto status = doUpgradeV2ToV3(kv); if (!status.ok()) { // rollback by snapshot return status; @@ -94,7 +116,7 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { } // static -Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { +Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { MetaDataUpgrade upgrader(kv); { // kSpacesTable @@ -105,7 +127,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { Status status = Status::OK(); while (iter->valid()) { if (FLAGS_print_info) { - upgrader.printSpaces(iter->val()); + upgrader.printSpacesV1(iter->val()); } status = upgrader.rewriteSpaces(iter->key(), iter->val()); if (!status.ok()) { @@ -158,6 +180,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kLeadersTable auto prefix = nebula::meta::v1::kLeadersTable; @@ -178,6 +201,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kTagsTable auto prefix = nebula::meta::v1::kTagsTable; @@ -327,7 +351,74 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } - if (!setMetaVersionToKV(kv)) { + if (!setMetaVersionToKV(kv, MetaVersion::V2)) { + return Status::Error("Persist meta version failed"); + } else { + return Status::OK(); + } +} + +Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { + MetaDataUpgrade upgrader(kv); + // Step 1: Upgrade HeartBeat into machine list + { + const auto& prefix = MetaKeyUtils::hostPrefix(); + std::unique_ptr iter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get active hosts failed"; + return Status::Error("Get hosts failed"); + } + + std::vector data; + while (iter->valid()) { + auto info = HostInfo::decode(iter->val()); + + if (info.role_ == meta::cpp2::HostRole::STORAGE) { + // Save the machine information + auto host = MetaKeyUtils::parseHostKey(iter->key()); + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + data.emplace_back(std::move(machineKey), ""); + + // Save the zone information + auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port); + auto zoneKey = MetaKeyUtils::zoneKey(std::move(zoneName)); + auto zoneVal = MetaKeyUtils::zoneVal({host}); + data.emplace_back(std::move(zoneKey), std::move(zoneVal)); + } + iter->next(); + } + auto status = upgrader.saveMachineAndZone(std::move(data)); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + } + + // Step 2: Update Create space properties about Group + { + const auto& prefix = MetaKeyUtils::spacePrefix(); + std::unique_ptr iter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get spaces failed"; + return Status::Error("Get spaces failed"); + } + + while (iter->valid()) { + if (FLAGS_print_info) { + upgrader.printSpacesV2(iter->val()); + } + auto spaceProperties = meta::v2::MetaServiceUtilsV2::parseSpace(iter->val()); + auto status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val()); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + iter->next(); + } + } + if (!setMetaVersionToKV(kv, MetaVersion::V3)) { return Status::Error("Persist meta version failed"); } else { return Status::OK(); diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index 59737175430..c6e31b8d01d 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -17,6 +17,7 @@ enum class MetaVersion { UNKNOWN = 0, V1 = 1, V2 = 2, + V3 = 3, }; /** @@ -28,17 +29,21 @@ class MetaVersionMan final { static MetaVersion getMetaVersionFromKV(kvstore::KVStore* kv); - static bool setMetaVersionToKV(kvstore::KVStore* kv); + static bool setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version); static Status updateMetaV1ToV2(kvstore::KVStore* kv); + static Status updateMetaV2ToV3(kvstore::KVStore* kv); + private: static MetaVersion getVersionByHost(kvstore::KVStore* kv); - static Status doUpgrade(kvstore::KVStore* kv); + static Status doUpgradeV1ToV2(kvstore::KVStore* kv); + + static Status doUpgradeV2ToV3(kvstore::KVStore* kv); }; } // namespace meta } // namespace nebula -#endif // META_ROOTUSERMAN_H_ +#endif // META_METAVERSIONMAN_H_ diff --git a/src/meta/processors/admin/ListClusterInfoProcessor.cpp b/src/meta/processors/admin/ListClusterInfoProcessor.cpp index ce5215e2f98..266284223bc 100644 --- a/src/meta/processors/admin/ListClusterInfoProcessor.cpp +++ b/src/meta/processors/admin/ListClusterInfoProcessor.cpp @@ -37,8 +37,8 @@ void ListClusterInfoProcessor::process(const cpp2::ListClusterInfoReq& req) { } auto iter = nebula::value(iterRet).get(); for (; iter->valid(); iter->next()) { - HostAddr addr = MetaKeyUtils::parseHostKey(iter->key()); - HostInfo info = HostInfo::decode(iter->val()); + auto addr = MetaKeyUtils::parseHostKey(iter->key()); + auto info = HostInfo::decode(iter->val()); cpp2::ServiceInfo service; service.role_ref() = info.role_; diff --git a/src/meta/processors/zone/MergeZoneProcessor.cpp b/src/meta/processors/zone/MergeZoneProcessor.cpp index 9a975f56408..fe8f1f227bd 100644 --- a/src/meta/processors/zone/MergeZoneProcessor.cpp +++ b/src/meta/processors/zone/MergeZoneProcessor.cpp @@ -151,7 +151,6 @@ void MergeZoneProcessor::process(const cpp2::MergeZoneReq& req) { for (auto& zone : zones) { auto it = std::find(spaceZones.begin(), spaceZones.end(), zone); if (it != spaceZones.end()) { - LOG(INFO) << "REMOVE ZONE " << zone; replacement = true; spaceZones.erase(it); } diff --git a/src/meta/processors/zone/SplitZoneProcessor.cpp b/src/meta/processors/zone/SplitZoneProcessor.cpp deleted file mode 100644 index 833091aeb8b..00000000000 --- a/src/meta/processors/zone/SplitZoneProcessor.cpp +++ /dev/null @@ -1,186 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/SplitZoneProcessor.h" - -namespace nebula { -namespace meta { - -void SplitZoneProcessor::process(const cpp2::SplitZoneReq& req) { - auto zoneName = req.get_zone_name(); - auto zoneKey = MetaKeyUtils::zoneKey(zoneName); - auto zoneValueRet = doGet(std::move(zoneKey)); - if (!nebula::ok(zoneValueRet)) { - LOG(ERROR) << "Zone " << zoneName << " not existed error: " - << apache::thrift::util::enumNameSafe(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND); - handleErrorCode(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND); - onFinished(); - return; - } - - auto oneZoneName = req.get_one_zone_name(); - auto oneZoneValueRet = doGet(MetaKeyUtils::zoneKey(oneZoneName)); - if (nebula::ok(oneZoneValueRet)) { - LOG(ERROR) << "Zone " << oneZoneName << " have existed"; - handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); - onFinished(); - return; - } - - auto anotherZoneName = req.get_another_zone_name(); - auto anotherZoneValueRet = doGet(MetaKeyUtils::zoneKey(anotherZoneName)); - if (nebula::ok(anotherZoneValueRet)) { - LOG(ERROR) << "Zone " << anotherZoneName << " have existed"; - handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); - onFinished(); - return; - } - - auto oneZoneHosts = req.get_one_zone_hosts(); - // Confirm that there are no duplicates in the parameters. - if (std::unique(oneZoneHosts.begin(), oneZoneHosts.end()) != oneZoneHosts.end()) { - LOG(ERROR) << "Zones have duplicated element"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - auto anotherZoneHosts = req.get_another_zone_hosts(); - // Confirm that there are no duplicates in the parameters. - if (std::unique(anotherZoneHosts.begin(), anotherZoneHosts.end()) != anotherZoneHosts.end()) { - LOG(ERROR) << "Zones have duplicated element"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - if (oneZoneHosts.empty() || anotherZoneHosts.empty()) { - LOG(ERROR) << "Hosts should not be empty"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; - for (auto& host : oneZoneHosts) { - auto iter = std::find(anotherZoneHosts.begin(), anotherZoneHosts.end(), host); - if (iter != anotherZoneHosts.end()) { - LOG(ERROR) << "Host " << host << " repeat with another zone hosts"; - code = nebula::cpp2::ErrorCode::E_CONFLICT; - break; - } - } - - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code); - onFinished(); - return; - } - - std::vector unionHosts; - std::sort(oneZoneHosts.begin(), oneZoneHosts.end()); - std::sort(anotherZoneHosts.begin(), anotherZoneHosts.end()); - std::set_union(oneZoneHosts.begin(), - oneZoneHosts.end(), - anotherZoneHosts.begin(), - anotherZoneHosts.end(), - std::back_inserter(unionHosts)); - - auto zoneHosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); - if (unionHosts.size() != zoneHosts.size()) { - LOG(ERROR) << "The total host is not all hosts"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - for (auto& host : unionHosts) { - auto iter = std::find(zoneHosts.begin(), zoneHosts.end(), host); - if (iter == zoneHosts.end()) { - LOG(ERROR) << "Host " << host << " not exist in original zone"; - code = nebula::cpp2::ErrorCode::E_INVALID_PARM; - break; - } - } - - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code); - onFinished(); - return; - } - - // Remove original zone - folly::Baton baton; - std::vector keys = {zoneKey}; - auto result = nebula::cpp2::ErrorCode::SUCCEEDED; - kvstore_->asyncMultiRemove(kDefaultSpaceId, - kDefaultPartId, - std::move(keys), - [&result, &baton](nebula::cpp2::ErrorCode res) { - if (nebula::cpp2::ErrorCode::SUCCEEDED != res) { - result = res; - LOG(ERROR) << "Remove data error on meta server"; - } - baton.post(); - }); - baton.wait(); - if (result != nebula::cpp2::ErrorCode::SUCCEEDED) { - this->handleErrorCode(result); - this->onFinished(); - return; - } - - std::vector data; - code = updateSpacesZone(data, zoneName, oneZoneName, anotherZoneName); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code); - onFinished(); - return; - } - - auto oneZoneKey = MetaKeyUtils::zoneKey(std::move(oneZoneName)); - auto oneZoneVal = MetaKeyUtils::zoneVal(std::move(oneZoneHosts)); - data.emplace_back(std::move(oneZoneKey), std::move(oneZoneVal)); - - auto anotherZoneKey = MetaKeyUtils::zoneKey(std::move(anotherZoneName)); - auto anotherZoneVal = MetaKeyUtils::zoneVal(std::move(anotherZoneHosts)); - data.emplace_back(std::move(anotherZoneKey), std::move(anotherZoneVal)); - doSyncPutAndUpdate(std::move(data)); -} - -nebula::cpp2::ErrorCode SplitZoneProcessor::updateSpacesZone(std::vector& data, - const std::string& originalZoneName, - const std::string& oneZoneName, - const std::string& anotherZoneName) { - const auto& prefix = MetaKeyUtils::spacePrefix(); - auto ret = doPrefix(prefix); - - if (!nebula::ok(ret)) { - LOG(ERROR) << "List spaces failed"; - return nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND; - } - - auto iter = nebula::value(ret).get(); - while (iter->valid()) { - auto spaceKey = iter->key(); - auto properties = MetaKeyUtils::parseSpace(iter->val()); - auto zones = properties.get_zone_names(); - - auto it = std::find(zones.begin(), zones.end(), originalZoneName); - if (it != zones.end()) { - zones.erase(it); - zones.emplace_back(oneZoneName); - zones.emplace_back(anotherZoneName); - properties.set_zone_names(zones); - auto spaceVal = MetaKeyUtils::spaceVal(properties); - data.emplace_back(spaceKey, std::move(spaceVal)); - } - iter->next(); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/SplitZoneProcessor.h b/src/meta/processors/zone/SplitZoneProcessor.h deleted file mode 100644 index 280bec98951..00000000000 --- a/src/meta/processors/zone/SplitZoneProcessor.h +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_SPLITZONEPROCESSOR_H -#define META_SPLITZONEPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class SplitZoneProcessor : public BaseProcessor { - public: - static SplitZoneProcessor* instance(kvstore::KVStore* kvstore) { - return new SplitZoneProcessor(kvstore); - } - - void process(const cpp2::SplitZoneReq& req); - - private: - explicit SplitZoneProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - - nebula::cpp2::ErrorCode updateSpacesZone(std::vector& data, - const std::string& originalZoneName, - const std::string& oneZoneName, - const std::string& anotherZoneName); -}; - -} // namespace meta -} // namespace nebula - -#endif // META_SPLITZONEPROCESSOR_H diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index 2a34f0fa766..dc7315aa395 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -2752,1008 +2752,6 @@ TEST(MetaClientTest, RocksdbOptionsTest) { client->createSpace(spaceDesc).get(); sleep(FLAGS_heartbeat_interval_secs + 1); } - { - auto listRet = client->listListener(space).get(); - ASSERT_TRUE(listRet.ok()) << listRet.status(); - auto listeners = listRet.value(); - ASSERT_EQ(0, listeners.size()); - } - cluster.stop(); -} - -TEST(MetaClientTest, VerifyClientTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/VerifyClientTest.XXXXXX"); - - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - - FLAGS_enable_client_white_list = true; - { - FLAGS_client_white_list = nebula::cpp2::common_constants::version(); - auto status = client->verifyVersion(); - EXPECT_TRUE(status.ok()); - } - { - FLAGS_client_white_list = ""; - auto status = client->verifyVersion(); - EXPECT_FALSE(status.ok()); - } - { - FLAGS_client_white_list = "1.0.0:1.2.0:"; - auto status = client->verifyVersion(); - EXPECT_FALSE(status.ok()); - } - { - FLAGS_enable_client_white_list = false; - FLAGS_client_white_list = "1.0.0:1.2.0:"; - auto status = client->verifyVersion(); - EXPECT_TRUE(status.ok()); - } - cluster.stop(); -} - -TEST(MetaClientTest, HostsTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/HostsTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - { - // Add single host - std::vector hosts = {{"127.0.0.1", 8989}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - { - // Add multi host - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - { - // Add duplicated hosts - std::vector hosts = {{"127.0.0.1", 8986}, {"127.0.0.1", 8986}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add empty hosts - std::vector hosts = {}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add hosts which is existed - std::vector hosts = {{"127.0.0.1", 8988}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add hosts which is existed - std::vector hosts = {{"127.0.0.1", 8986}, {"127.0.0.1", 8988}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Show the zones created by add hosts - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - ASSERT_EQ(3, result.value().size()); - } - { - // Drop hosts with duplicate element - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8987}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Drop hosts which is empty - std::vector hosts = {}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - cluster.stop(); -} - -TEST(MetaClientTest, AddHostsIntoNewZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/AddHostsIntoNewZoneTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - { - std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - { - // Add host into zone with duplicate hosts - std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add host into new zone with empty hosts. - std::vector hosts = {}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add host into new zone. - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_TRUE(result.ok()); - } - { - // Add host into new zone with zone name conflict. - std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_FALSE(result.ok()); - } - { - // the hosts have exist in another zones. - std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); - EXPECT_FALSE(result.ok()); - } - cluster.stop(); -} - -TEST(MetaClientTest, AddHostsIntoZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/AddHostsIntoZoneTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - { - // Add host into zone with duplicate hosts - std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", false).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add host into zone with empty hosts - std::vector hosts; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", false).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add host into zone which zone is not exist. - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_not_existed", false).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add host into zone. - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_TRUE(result.ok()); - } - { - // Add host into zone with zone name conflict. - std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_FALSE(result.ok()); - } - { - // Add existed hosts - std::vector hosts = {{"127.0.0.1", 8988}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_FALSE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); - EXPECT_TRUE(result.ok()); - } - { - // Add existed hosts. - std::vector hosts = {{"127.0.0.1", 8988}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8988}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", false).get(); - EXPECT_FALSE(result.ok()); - } - { - // Drop hosts which is empty. - std::vector hosts = {}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Drop hosts which have duplicate element. - std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Drop hosts. - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - cluster.stop(); -} - -TEST(MetaClientTest, DropHostsTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/DropHostsTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - auto* kv = cluster.metaKV_.get(); - { - std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB(kv, {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(3, zones.size()); - ASSERT_EQ("default_zone_127.0.0.1_8987", zones[0].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8988", zones[1].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8989", zones[2].get_zone_name()); - } - { - // Create Space on cluster, the replica number same with the zone size. - meta::cpp2::SpaceDesc properties; - properties.set_space_name("default_space_0"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(1, ret.value()); - } - { - // Create Space on cluster, the replica number less than the zone size. - cpp2::SpaceDesc properties; - properties.set_space_name("default_space_1"); - properties.set_partition_num(9); - properties.set_replica_factor(1); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(2, ret.value()); - } - { - // Create Space on cluster, the replica number greater than the zone size. - cpp2::SpaceDesc properties; - properties.set_space_name("default_space_2"); - properties.set_partition_num(9); - properties.set_replica_factor(6); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - auto ret = client->createSpace(properties).get(); - ASSERT_FALSE(ret.ok()) << ret.status(); - } - { - std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8978}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8979}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_2", true).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB( - kv, {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(6, zones.size()); - ASSERT_EQ("default_zone_127.0.0.1_8987", zones[0].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8988", zones[1].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8989", zones[2].get_zone_name()); - ASSERT_EQ("zone_0", zones[3].get_zone_name()); - ASSERT_EQ("zone_1", zones[4].get_zone_name()); - ASSERT_EQ("zone_2", zones[5].get_zone_name()); - } - { - // Create Space on cluster, the replica number greater than the zone size. - cpp2::SpaceDesc properties; - properties.set_space_name("default_space_on_zone_3"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - properties.set_zone_names(std::move(zones)); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(4, ret.value()); - } - { - // Create Space on cluster, the replica number less than the zone size - cpp2::SpaceDesc properties; - properties.set_space_name("default_space_on_zone_1"); - properties.set_partition_num(9); - properties.set_replica_factor(1); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - std::vector zones = {"zone_0"}; - properties.set_zone_names(std::move(zones)); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(5, ret.value()); - } - { - // Create Space on cluster, the replica number greater than the zone size - cpp2::SpaceDesc properties; - properties.set_space_name("default_space_on_zone_6"); - properties.set_partition_num(9); - properties.set_replica_factor(6); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - std::vector zones = {"zone_0"}; - properties.set_zone_names(std::move(zones)); - auto ret = client->createSpace(properties).get(); - ASSERT_FALSE(ret.ok()) << ret.status(); - } - { - // Drop hosts which hold partition. - std::vector hosts = {{"127.0.0.1", 8987}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - // Drop hosts which hold partition. - std::vector hosts = {{"127.0.0.1", 8977}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_FALSE(result.ok()); - } - { - auto ret = client->dropSpace("default_space_on_zone_1").get(); - ASSERT_TRUE(ret.ok()); - } - { - auto ret = client->dropSpace("default_space_on_zone_3").get(); - ASSERT_TRUE(ret.ok()); - } - { - auto ret = client->dropSpace("default_space_0").get(); - ASSERT_TRUE(ret.ok()); - } - { - auto ret = client->dropSpace("default_space_1").get(); - ASSERT_TRUE(ret.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8987}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8976}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8978}}; - auto result = client->dropHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(4, zones.size()); - ASSERT_EQ("default_zone_127.0.0.1_8988", zones[0].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8989", zones[1].get_zone_name()); - ASSERT_EQ("zone_0", zones[2].get_zone_name()); - ASSERT_EQ("zone_2", zones[3].get_zone_name()); - } - cluster.stop(); -} - -TEST(MetaClientTest, RenameZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/RenameZoneTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - auto* kv = cluster.metaKV_.get(); - { - std::vector hosts = {{"127.0.0.1", 8987}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8988}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_2", true).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB(kv, {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(3, zones.size()); - ASSERT_EQ("zone_0", zones[0].get_zone_name()); - ASSERT_EQ("zone_1", zones[1].get_zone_name()); - ASSERT_EQ("zone_2", zones[2].get_zone_name()); - } - { - cpp2::SpaceDesc properties; - properties.set_space_name("default"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - properties.set_zone_names(std::move(zones)); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(1, ret.value()); - } - { - auto ret = client->getSpace("default").get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - auto spaceDesc = ret.value().get_properties(); - ASSERT_EQ("default", spaceDesc.get_space_name()); - ASSERT_EQ(9, spaceDesc.get_partition_num()); - ASSERT_EQ(3, spaceDesc.get_replica_factor()); - ASSERT_EQ("utf8", spaceDesc.get_charset_name()); - ASSERT_EQ("utf8_bin", spaceDesc.get_collate_name()); - auto zones = spaceDesc.get_zone_names(); - ASSERT_EQ(3, zones.size()); - ASSERT_EQ("zone_0", zones[0]); - ASSERT_EQ("zone_1", zones[1]); - ASSERT_EQ("zone_2", zones[2]); - } - { - auto result = client->renameZone("zone_not_exist", "new_zone_name").get(); - EXPECT_FALSE(result.ok()); - } - { - auto result = client->renameZone("zone_0", "zone_1").get(); - EXPECT_FALSE(result.ok()); - } - { - auto result = client->renameZone("zone_1", "z_1").get(); - EXPECT_TRUE(result.ok()); - } - { - auto ret = client->getSpace("default").get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - auto spaceDesc = ret.value().get_properties(); - ASSERT_EQ("default", spaceDesc.get_space_name()); - ASSERT_EQ(9, spaceDesc.get_partition_num()); - ASSERT_EQ(3, spaceDesc.get_replica_factor()); - ASSERT_EQ("utf8", spaceDesc.get_charset_name()); - ASSERT_EQ("utf8_bin", spaceDesc.get_collate_name()); - auto zones = spaceDesc.get_zone_names(); - ASSERT_EQ(3, zones.size()); - ASSERT_EQ("zone_0", zones[0]); - ASSERT_EQ("z_1", zones[1]); - ASSERT_EQ("zone_2", zones[2]); - } - cluster.stop(); -} - -TEST(MetaClientTest, MergeZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/MergeZoneTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - auto* kv = cluster.metaKV_.get(); - { - std::vector hosts = { - {"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB( - kv, {{"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(4, zones.size()); - ASSERT_EQ("default_zone_127.0.0.1_8986", zones[0].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8987", zones[1].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8988", zones[2].get_zone_name()); - ASSERT_EQ("default_zone_127.0.0.1_8989", zones[3].get_zone_name()); - } - { - // Merge zones is empty - auto result = client->mergeZone({}, "new_zone").get(); - EXPECT_FALSE(result.ok()); - } - { - // Merge only zone - auto result = client->mergeZone({"zone_0"}, "new_zone").get(); - EXPECT_FALSE(result.ok()); - } - { - // Merge zones have duplicate - auto result = client->mergeZone({"zone_0", "zone_0"}, "new_zone").get(); - EXPECT_FALSE(result.ok()); - } - { - // Merge zones not exist - auto result = client->mergeZone({"zone_0", "zone_not_exist"}, "new_zone").get(); - EXPECT_FALSE(result.ok()); - } - { - // Merge zones not exist - auto result = client->mergeZone({"zone_not_exist_0", "zone_not_exist_1"}, "new_zone").get(); - EXPECT_FALSE(result.ok()); - } - { - auto result = client - ->mergeZone({"default_zone_127.0.0.1_8986", - "default_zone_127.0.0.1_8987", - "default_zone_127.0.0.1_8988"}, - "zone_1") - .get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->mergeZone({"default_zone_127.0.0.1_8989", "zone_1"}, "zone_1").get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(1, zones.size()); - ASSERT_EQ("zone_1", zones[0].get_zone_name()); - } - { - std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB(kv, {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}}); - // Merge zone with space test - { - cpp2::SpaceDesc properties; - properties.set_space_name("default_space"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - std::vector zones = {"default_zone_127.0.0.1_8976", - "default_zone_127.0.0.1_8977", - "default_zone_127.0.0.1_8978"}; - properties.set_zone_names(std::move(zones)); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(1, ret.value()); - } - { - // zone size should be greater than part size - auto result = - client->mergeZone({"default_zone_127.0.0.1_8976", "default_zone_127.0.0.1_8977"}, "zone") - .get(); - EXPECT_FALSE(result.ok()); - } - { - auto result = client->dropSpace("default_space").get(); - EXPECT_TRUE(result.ok()); - } - { - cpp2::SpaceDesc properties; - properties.set_space_name("default_space"); - properties.set_partition_num(9); - properties.set_replica_factor(1); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - std::vector zones = {"default_zone_127.0.0.1_8976", - "default_zone_127.0.0.1_8977", - "default_zone_127.0.0.1_8978"}; - properties.set_zone_names(std::move(zones)); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(2, ret.value()); - } - { - auto result = - client->mergeZone({"default_zone_127.0.0.1_8976", "default_zone_127.0.0.1_8977"}, "z_1") - .get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->mergeZone({"default_zone_127.0.0.1_8978", "z_1"}, "z_1").get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->getSpace("default_space").get(); - auto properties = result.value().get_properties(); - ASSERT_EQ("default_space", properties.get_space_name()); - ASSERT_EQ(9, properties.get_partition_num()); - ASSERT_EQ(1, properties.get_replica_factor()); - ASSERT_EQ("utf8", properties.get_charset_name()); - ASSERT_EQ("utf8_bin", properties.get_collate_name()); - auto zones = properties.get_zone_names(); - ASSERT_EQ(1, zones.size()); - ASSERT_EQ("z_1", zones[0]); - } - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(2, zones.size()); - ASSERT_EQ("z_1", zones[0].get_zone_name()); - ASSERT_EQ("zone_1", zones[1].get_zone_name()); - } - cluster.stop(); -} - -TEST(MetaClientTest, SplitZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/SplitZoneTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - auto* kv = cluster.metaKV_.get(); - { - std::vector hosts = { - {"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "default_zone", true).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB( - kv, {{"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(1, zones.size()); - ASSERT_EQ("default_zone", zones[0].get_zone_name()); - } - { - // Split zone which not exist - auto result = client - ->splitZone("zone_not_exist", - "one_zone", - {{"127.0.0.1", 8986}, {"127.0.0.1", 8987}}, - "another_zone", - {{"127.0.0.1", 8988}, {"127.0.0.1", 8989}}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - // Split zone with empty hosts - auto result = client - ->splitZone("default_zone", - "one_zone", - {}, - "another_zone", - {{"127.0.0.1", 8988}, {"127.0.0.1", 8989}}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - // Split zone with empty hosts - auto result = client - ->splitZone("default_zone", - "one_zone", - {{"127.0.0.1", 8986}, {"127.0.0.1", 8987}}, - "another_zone", - {}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - // Split zone and the sum is not all - auto result = client - ->splitZone("default_zone", - "one_zone", - {{"127.0.0.1", 8986}}, - "another_zone", - {{"127.0.0.1", 8988}, {"127.0.0.1", 8989}}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - // Split zone and the hosts is more than the total - auto result = client - ->splitZone("default_zone", - "one_zone", - {{"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8985}}, - "another_zone", - {{"127.0.0.1", 8988}, {"127.0.0.1", 8989}}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - // Split empty zone successfully - auto result = client - ->splitZone("default_zone", - "one_zone", - {{{"127.0.0.1", 8986}, {"127.0.0.1", 8987}}}, - "another_zone", - {{"127.0.0.1", 8988}, {"127.0.0.1", 8989}}) - .get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(2, zones.size()); - ASSERT_EQ("another_zone", zones[0].get_zone_name()); - ASSERT_EQ("one_zone", zones[1].get_zone_name()); - } - { - std::vector hosts = { - {"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; - auto result = client->addHostsIntoZone(std::move(hosts), "default_zone", true).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB( - kv, {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(3, zones.size()); - ASSERT_EQ("another_zone", zones[0].get_zone_name()); - ASSERT_EQ("default_zone", zones[1].get_zone_name()); - ASSERT_EQ("one_zone", zones[2].get_zone_name()); - } - { - cpp2::SpaceDesc properties; - properties.set_space_name("default_space"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(1, ret.value()); - } - { - auto ret = client->getSpace("default_space").get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - auto spaceDesc = ret.value().get_properties(); - ASSERT_EQ("default_space", spaceDesc.get_space_name()); - ASSERT_EQ(9, spaceDesc.get_partition_num()); - ASSERT_EQ(3, spaceDesc.get_replica_factor()); - ASSERT_EQ("utf8", spaceDesc.get_charset_name()); - ASSERT_EQ("utf8_bin", spaceDesc.get_collate_name()); - auto zones = spaceDesc.get_zone_names(); - ASSERT_EQ(3, zones.size()); - ASSERT_EQ("another_zone", zones[0]); - ASSERT_EQ("default_zone", zones[1]); - ASSERT_EQ("one_zone", zones[2]); - } - { - // Zone name conflict - auto result = client - ->splitZone("default_zone", - "one_zone", - {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}}, - "another_zone_1", - {{"127.0.0.1", 8978}, {"127.0.0.1", 8979}}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - // Zone name conflict - auto result = client - ->splitZone("default_zone", - "one_zone_1", - {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}}, - "another_zone", - {{"127.0.0.1", 8978}, {"127.0.0.1", 8979}}) - .get(); - EXPECT_FALSE(result.ok()); - } - { - auto result = client - ->splitZone("default_zone", - "one_zone_1", - {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}}, - "another_zone_1", - {{"127.0.0.1", 8978}, {"127.0.0.1", 8979}}) - .get(); - EXPECT_TRUE(result.ok()); - } - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(4, zones.size()); - ASSERT_EQ("another_zone", zones[0].get_zone_name()); - ASSERT_EQ("another_zone_1", zones[1].get_zone_name()); - ASSERT_EQ("one_zone", zones[2].get_zone_name()); - ASSERT_EQ("one_zone_1", zones[3].get_zone_name()); - } - { - auto ret = client->getSpace("default_space").get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - auto spaceDesc = ret.value().get_properties(); - ASSERT_EQ("default_space", spaceDesc.get_space_name()); - ASSERT_EQ(9, spaceDesc.get_partition_num()); - ASSERT_EQ(3, spaceDesc.get_replica_factor()); - ASSERT_EQ("utf8", spaceDesc.get_charset_name()); - ASSERT_EQ("utf8_bin", spaceDesc.get_collate_name()); - auto zones = spaceDesc.get_zone_names(); - ASSERT_EQ(4, zones.size()); - ASSERT_EQ("another_zone", zones[0]); - ASSERT_EQ("one_zone", zones[1]); - ASSERT_EQ("one_zone_1", zones[2]); - ASSERT_EQ("another_zone_1", zones[3]); - } - cluster.stop(); -} - -TEST(MetaClientTest, DropZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/DropZoneTest.XXXXXX"); - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); - auto* kv = cluster.metaKV_.get(); - { - // Add single host - std::vector hosts = {{"127.0.0.1", 8986}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8987}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8988}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_2", true).get(); - EXPECT_TRUE(result.ok()); - } - { - std::vector hosts = {{"127.0.0.1", 8989}}; - auto result = client->addHostsIntoZone(std::move(hosts), "zone_3", true).get(); - EXPECT_TRUE(result.ok()); - } - TestUtils::registerHB( - kv, {{"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - auto zones = result.value(); - ASSERT_EQ(4, zones.size()); - ASSERT_EQ("zone_0", zones[0].get_zone_name()); - ASSERT_EQ("zone_1", zones[1].get_zone_name()); - ASSERT_EQ("zone_2", zones[2].get_zone_name()); - ASSERT_EQ("zone_3", zones[3].get_zone_name()); - } - { - auto result = client->dropZone("zone_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - { - auto result = client->dropZone("zone_0").get(); - ASSERT_TRUE(result.ok()); - } - { - auto result = client->dropZone("zone_0").get(); - ASSERT_FALSE(result.ok()); - } - { - cpp2::SpaceDesc properties; - properties.set_space_name("default_space"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(1, ret.value()); - } - { - auto result = client->dropZone("zone_1").get(); - ASSERT_FALSE(result.ok()); - } - { - auto result = client->dropSpace("default_space").get(); - ASSERT_TRUE(result.ok()); - } - { - cpp2::SpaceDesc properties; - properties.set_space_name("default_space"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_charset_name("utf8"); - properties.set_collate_name("utf8_bin"); - auto ret = client->createSpace(properties).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - ASSERT_EQ(2, ret.value()); - } - { - auto result = client->dropZone("zone_1").get(); - ASSERT_FALSE(result.ok()); - } - cluster.stop(); -} - -TEST(MetaClientTest, RocksdbOptionsTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/RocksdbOptionsTest.XXXXXX"); - - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - // auto* kv = cluster.metaKV_.get(); - // TestUtils::createSomeHosts(kv, {{"0", 0}}); - - MetaClientOptions options; - // Now the `--local_config' option only affect if initialize the configuration - // from meta not affect `show/update/get configs' - options.skipConfig_ = false; - - cluster.initMetaClient(std::move(options)); - auto* client = cluster.metaClient_.get(); - { - // Add single host - std::vector hosts = {{"0", 0}}; - auto result = client->addHosts(std::move(hosts)).get(); - EXPECT_TRUE(result.ok()); - } - - auto listener = std::make_unique(); - auto module = cpp2::ConfigModule::STORAGE; - auto mode = meta::cpp2::ConfigMode::MUTABLE; - - client->registerListener(listener.get()); - client->gflagsModule_ = module; - - // mock some rocksdb gflags to meta - { - auto name = "rocksdb_db_options"; - std::vector configItems; - FLAGS_rocksdb_db_options = - R"({"disable_auto_compactions":"false","write_buffer_size":"1048576"})"; - Map map; - map.kvs.emplace("disable_auto_compactions", "false"); - map.kvs.emplace("write_buffer_size", "1048576"); - configItems.emplace_back(initConfigItem(module, name, mode, Value(map))); - client->regConfig(configItems); - } - { - std::vector hosts = {{"0", 0}}; - TestUtils::registerHB(cluster.metaKV_.get(), hosts); - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("default_space"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(1); - client->createSpace(spaceDesc).get(); - sleep(FLAGS_heartbeat_interval_secs + 1); - } { std::string name = "rocksdb_db_options"; Map map; diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index 77505dab69c..b5723d1f231 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -39,7 +39,6 @@ #include "meta/processors/zone/ListZonesProcessor.h" #include "meta/processors/zone/MergeZoneProcessor.h" #include "meta/processors/zone/RenameZoneProcessor.h" -#include "meta/processors/zone/SplitZoneProcessor.h" #include "meta/test/TestUtils.h" DECLARE_int32(expired_threshold_sec); diff --git a/src/meta/upgrade/CMakeLists.txt b/src/meta/upgrade/CMakeLists.txt index 7b895c31e36..b0700952b86 100644 --- a/src/meta/upgrade/CMakeLists.txt +++ b/src/meta/upgrade/CMakeLists.txt @@ -6,11 +6,14 @@ nebula_add_library( meta_data_upgrade_obj OBJECT MetaDataUpgrade.cpp v1/MetaServiceUtilsV1.cpp + v2/MetaServiceUtilsV2.cpp ) add_dependencies( meta_data_upgrade_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) nebula_add_subdirectory(v1) +nebula_add_subdirectory(v2) diff --git a/src/meta/upgrade/MetaDataUpgrade.cpp b/src/meta/upgrade/MetaDataUpgrade.cpp index 8bd89806738..e057e8d6f85 100644 --- a/src/meta/upgrade/MetaDataUpgrade.cpp +++ b/src/meta/upgrade/MetaDataUpgrade.cpp @@ -19,6 +19,7 @@ #include "meta/ActiveHostsMan.h" #include "meta/MetaServiceUtils.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DECLARE_bool(null_type); DECLARE_uint32(string_index_limit); @@ -62,6 +63,68 @@ Status MetaDataUpgrade::rewriteSpaces(const folly::StringPiece &key, return Status::OK(); } +Status MetaDataUpgrade::rewriteSpacesV2ToV3(const folly::StringPiece &key, + const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + cpp2::SpaceDesc spaceDesc; + spaceDesc.space_name_ref() = oldProps.get_space_name(); + spaceDesc.partition_num_ref() = oldProps.get_partition_num(); + spaceDesc.replica_factor_ref() = oldProps.get_replica_factor(); + spaceDesc.charset_name_ref() = oldProps.get_charset_name(); + spaceDesc.collate_name_ref() = oldProps.get_collate_name(); + cpp2::ColumnTypeDef def; + auto &type = oldProps.get_vid_type(); + def.type_length_ref() = *type.get_type_length(); + def.type_ref() = convertToPropertyType(type.get_type()); + + if (type.geo_shape_ref().has_value()) { + def.geo_shape_ref() = convertToGeoShape(*type.get_geo_shape()); + } + spaceDesc.vid_type_ref() = std::move(def); + if (oldProps.isolation_level_ref().has_value()) { + if (*oldProps.isolation_level_ref() == nebula::meta::v2::cpp2::IsolationLevel::DEFAULT) { + spaceDesc.isolation_level_ref() = nebula::meta::cpp2::IsolationLevel::DEFAULT; + } else { + spaceDesc.isolation_level_ref() = nebula::meta::cpp2::IsolationLevel::TOSS; + } + } + + if (oldProps.comment_ref().has_value()) { + spaceDesc.comment_ref() = *oldProps.comment_ref(); + } + + if (oldProps.group_name_ref().has_value()) { + auto groupName = *oldProps.group_name_ref(); + auto groupKey = meta::v2::MetaServiceUtilsV2::groupKey(groupName); + std::string zoneValue; + auto code = kv_->get(kDefaultSpaceId, kDefaultPartId, std::move(groupKey), &zoneValue); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Group Failed"); + } + + auto zones = meta::v2::MetaServiceUtilsV2::parseZoneNames(std::move(zoneValue)); + spaceDesc.zone_names_ref() = std::move(zones); + } else { + const auto &zonePrefix = MetaKeyUtils::zonePrefix(); + std::unique_ptr iter; + auto code = kv_->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Zones Failed"); + } + + std::vector<::std::string> zones; + while (iter->valid()) { + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + zones.emplace_back(std::move(zoneName)); + iter->next(); + } + spaceDesc.zone_names_ref() = std::move(zones); + } + + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::spaceVal(spaceDesc))); + return Status::OK(); +} + Status MetaDataUpgrade::rewriteParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto oldHosts = meta::v1::MetaServiceUtilsV1::parsePartVal(val); std::vector newHosts; @@ -278,6 +341,60 @@ Status MetaDataUpgrade::convertToNewIndexColumns( return Status::OK(); } +nebula::cpp2::PropertyType MetaDataUpgrade::convertToPropertyType( + nebula::meta::v2::cpp2::PropertyType type) { + switch (type) { + case nebula::meta::v2::cpp2::PropertyType::BOOL: + return nebula::cpp2::PropertyType::BOOL; + case nebula::meta::v2::cpp2::PropertyType::INT64: + return nebula::cpp2::PropertyType::INT64; + case nebula::meta::v2::cpp2::PropertyType::VID: + return nebula::cpp2::PropertyType::VID; + case nebula::meta::v2::cpp2::PropertyType::FLOAT: + return nebula::cpp2::PropertyType::FLOAT; + case nebula::meta::v2::cpp2::PropertyType::DOUBLE: + return nebula::cpp2::PropertyType::DOUBLE; + case nebula::meta::v2::cpp2::PropertyType::STRING: + return nebula::cpp2::PropertyType::STRING; + case nebula::meta::v2::cpp2::PropertyType::FIXED_STRING: + return nebula::cpp2::PropertyType::FIXED_STRING; + case nebula::meta::v2::cpp2::PropertyType::INT8: + return nebula::cpp2::PropertyType::INT8; + case nebula::meta::v2::cpp2::PropertyType::INT16: + return nebula::cpp2::PropertyType::INT16; + case nebula::meta::v2::cpp2::PropertyType::INT32: + return nebula::cpp2::PropertyType::INT32; + case nebula::meta::v2::cpp2::PropertyType::TIMESTAMP: + return nebula::cpp2::PropertyType::TIMESTAMP; + case nebula::meta::v2::cpp2::PropertyType::DATE: + return nebula::cpp2::PropertyType::DATE; + case nebula::meta::v2::cpp2::PropertyType::DATETIME: + return nebula::cpp2::PropertyType::DATETIME; + case nebula::meta::v2::cpp2::PropertyType::TIME: + return nebula::cpp2::PropertyType::TIME; + case nebula::meta::v2::cpp2::PropertyType::GEOGRAPHY: + return nebula::cpp2::PropertyType::GEOGRAPHY; + default: + return nebula::cpp2::PropertyType::UNKNOWN; + } +} + +nebula::meta::cpp2::GeoShape MetaDataUpgrade::convertToGeoShape( + nebula::meta::v2::cpp2::GeoShape shape) { + switch (shape) { + case nebula::meta::v2::cpp2::GeoShape::ANY: + return nebula::meta::cpp2::GeoShape::ANY; + case nebula::meta::v2::cpp2::GeoShape::POINT: + return nebula::meta::cpp2::GeoShape::POINT; + case nebula::meta::v2::cpp2::GeoShape::LINESTRING: + return nebula::meta::cpp2::GeoShape::LINESTRING; + case nebula::meta::v2::cpp2::GeoShape::POLYGON: + return nebula::meta::cpp2::GeoShape::POLYGON; + default: + LOG(FATAL) << "Unimplemented"; + } +} + void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::StringPiece &val) { auto host = meta::v1::MetaServiceUtilsV1::parseHostKey(key); auto info = HostInfo::decodeV1(val); @@ -288,7 +405,7 @@ void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::Stri LOG(INFO) << "Host info: gitInfoSha_: " << info.gitInfoSha_; } -void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { +void MetaDataUpgrade::printSpacesV1(const folly::StringPiece &val) { auto oldProps = meta::v1::MetaServiceUtilsV1::parseSpace(val); LOG(INFO) << "Space name: " << oldProps.get_space_name(); LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); @@ -297,6 +414,18 @@ void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); } +void MetaDataUpgrade::printSpacesV2(const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + LOG(INFO) << "Space name: " << oldProps.get_space_name(); + LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); + LOG(INFO) << "Replica factor: " << oldProps.get_replica_factor(); + LOG(INFO) << "Charset name: " << oldProps.get_charset_name(); + LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); + if (oldProps.group_name_ref().has_value()) { + LOG(INFO) << "Group name: " << *oldProps.group_name_ref(); + } +} + void MetaDataUpgrade::printParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto spaceId = meta::v1::MetaServiceUtilsV1::parsePartKeySpaceId(key); auto partId = meta::v1::MetaServiceUtilsV1::parsePartKeyPartId(key); @@ -434,5 +563,10 @@ void MetaDataUpgrade::printJobDesc(const folly::StringPiece &key, const folly::S LOG(INFO) << "JobDesc stopTime: " << stopTime; } +Status MetaDataUpgrade::saveMachineAndZone(std::vector data) { + NG_LOG_AND_RETURN_IF_ERROR(put(data)); + return Status::OK(); +} + } // namespace meta } // namespace nebula diff --git a/src/meta/upgrade/MetaDataUpgrade.h b/src/meta/upgrade/MetaDataUpgrade.h index dbea1afcbfc..bd4ffea08a1 100644 --- a/src/meta/upgrade/MetaDataUpgrade.h +++ b/src/meta/upgrade/MetaDataUpgrade.h @@ -14,6 +14,7 @@ #include "kvstore/KVStore.h" #include "meta/processors/Common.h" #include "meta/upgrade/v1/gen-cpp2/meta_types.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" namespace nebula { namespace meta { @@ -33,10 +34,15 @@ class MetaDataUpgrade final { Status rewriteConfigs(const folly::StringPiece &key, const folly::StringPiece &val); Status rewriteJobDesc(const folly::StringPiece &key, const folly::StringPiece &val); + Status rewriteSpacesV2ToV3(const folly::StringPiece &key, const folly::StringPiece &val); + Status deleteKeyVal(const folly::StringPiece &key); + Status saveMachineAndZone(std::vector data); + void printHost(const folly::StringPiece &key, const folly::StringPiece &val); - void printSpaces(const folly::StringPiece &val); + void printSpacesV1(const folly::StringPiece &val); + void printSpacesV2(const folly::StringPiece &val); void printParts(const folly::StringPiece &key, const folly::StringPiece &val); void printLeaders(const folly::StringPiece &key); void printSchemas(const folly::StringPiece &val); @@ -67,6 +73,26 @@ class MetaDataUpgrade final { return Status::OK(); } + Status put(std::vector data) { + folly::Baton baton; + auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; + kv_->asyncMultiPut(kDefaultSpaceId, + kDefaultPartId, + std::move(data), + [&ret, &baton](nebula::cpp2::ErrorCode code) { + if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { + ret = code; + LOG(INFO) << "Put data error on meta server"; + } + baton.post(); + }); + baton.wait(); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Put data failed"); + } + return Status::OK(); + } + Status remove(const folly::StringPiece &key) { std::vector keys{key.str()}; folly::Baton baton; @@ -94,6 +120,10 @@ class MetaDataUpgrade final { Status convertToNewIndexColumns(const std::vector &oldCols, std::vector &newCols); + nebula::cpp2::PropertyType convertToPropertyType(nebula::meta::v2::cpp2::PropertyType type); + + nebula::meta::cpp2::GeoShape convertToGeoShape(nebula::meta::v2::cpp2::GeoShape shape); + private: kvstore::KVStore *kv_ = nullptr; }; diff --git a/src/meta/upgrade/v2/CMakeLists.txt b/src/meta/upgrade/v2/CMakeLists.txt new file mode 100644 index 00000000000..df5d9d2549f --- /dev/null +++ b/src/meta/upgrade/v2/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +set(THRIFT1 ${Fbthrift_BIN}) + +set(meta_v2_sources + gen-cpp2/meta_constants.cpp + gen-cpp2/meta_data.cpp + gen-cpp2/meta_metadata.cpp + gen-cpp2/meta_types.cpp + ) + +add_custom_command( + OUTPUT ${meta_v2_sources} + COMMAND "${THRIFT1}" "--strict" "--allow-neg-enum-vals" "--gen" "mstch_cpp2:include_prefix=\"meta/upgrade/v2\",process_in_event_base,stack_arguments" "-o" "." "${CMAKE_CURRENT_SOURCE_DIR}/meta.thrift" + DEPENDS meta.thrift + ) + +nebula_add_library(meta_v2_thrift_obj OBJECT ${meta_v2_sources}) + +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-pedantic") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-extra") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-deprecated-declarations") diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp new file mode 100644 index 00000000000..bb23cd29c74 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" + +#include +#include + +namespace nebula::meta::v2 { + +std::string MetaServiceUtilsV2::groupKey(const std::string& group) { + std::string key; + key.reserve(kGroupsTable.size() + group.size()); + key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); + return key; +} + +std::vector MetaServiceUtilsV2::parseZoneNames(folly::StringPiece rawData) { + std::vector zones; + folly::split(',', rawData.str(), zones); + return zones; +} + +cpp2::SpaceDesc MetaServiceUtilsV2::parseSpace(folly::StringPiece rawData) { + cpp2::SpaceDesc spaceDesc; + apache::thrift::CompactSerializer::deserialize(rawData, spaceDesc); + return spaceDesc; +} + +} // namespace nebula::meta::v2 diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.h b/src/meta/upgrade/v2/MetaServiceUtilsV2.h new file mode 100644 index 00000000000..47bf010ce98 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ +#define TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ + +#include "common/base/Base.h" +#include "common/base/Status.h" +#include "common/thrift/ThriftTypes.h" +#include "interface/gen-cpp2/meta_types.h" +#include "kvstore/Common.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" + +namespace nebula::meta::v2 { + +const std::string kGroupsTable = "__groups__"; // NOLINT + +class MetaServiceUtilsV2 final { + public: + MetaServiceUtilsV2() = delete; + + static std::string groupKey(const std::string& group); + + static std::vector parseZoneNames(folly::StringPiece rawData); + + static cpp2::SpaceDesc parseSpace(folly::StringPiece rawData); +}; + +} // namespace nebula::meta::v2 + +#endif // TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ diff --git a/src/meta/upgrade/v2/meta.thrift b/src/meta/upgrade/v2/meta.thrift new file mode 100644 index 00000000000..d46922478b8 --- /dev/null +++ b/src/meta/upgrade/v2/meta.thrift @@ -0,0 +1,81 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +namespace cpp nebula.meta.v2 + +cpp_include "common/thrift/ThriftTypes.h" + +typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID +typedef i32 (cpp.type = "nebula::PartitionID") PartitionID +typedef i32 (cpp.type = "nebula::TagID") TagID +typedef i32 (cpp.type = "nebula::EdgeType") EdgeType +typedef i64 (cpp.type = "nebula::EdgeRanking") EdgeRanking +typedef i64 (cpp.type = "nebula::VertexID") VertexID +typedef i32 (cpp.type = "nebula::IndexID") IndexID + +typedef i32 IPv4 +typedef i32 (cpp.type = "nebula::Port") Port + +typedef i64 (cpp.type = "nebula::SchemaVer") SchemaVer + +struct SpaceDesc { + 1: binary space_name, + 2: i32 partition_num = 0, + 3: i32 replica_factor = 0, + 4: binary charset_name, + 5: binary collate_name, + 6: ColumnTypeDef vid_type = {"type": PropertyType.FIXED_STRING, "type_length": 8}, + 7: optional binary group_name, + 8: optional IsolationLevel isolation_level, + 9: optional binary comment, +} + +enum PropertyType { + UNKNOWN = 0, + + // Simple types + BOOL = 1, + INT64 = 2, // This is the same as INT in v1 + VID = 3, // Deprecated, only supported by v1 + FLOAT = 4, + DOUBLE = 5, + STRING = 6, + // String with fixed length. If the string content is shorteri + // than the given length, '\0' will be padded to the end + FIXED_STRING = 7, // New in v2 + INT8 = 8, // New in v2 + INT16 = 9, // New in v2 + INT32 = 10, // New in v2 + + // Date time + TIMESTAMP = 21, + DATE = 24, + DATETIME = 25, + TIME = 26, + + // Geo spatial + GEOGRAPHY = 31, +} (cpp.enum_strict) + +// Geo shape type +enum GeoShape { + ANY = 0, + POINT = 1, + LINESTRING = 2, + POLYGON = 3, +} (cpp.enum_strict) + +struct ColumnTypeDef { + 1: required PropertyType type, + // type_length is valid for fixed_string type + 2: optional i16 type_length = 0, + // geo_shape is valid for geography type + 3: optional GeoShape geo_shape, +} + +enum IsolationLevel { + DEFAULT = 0x00, // allow add half edge(either in or out edge succeeded) + TOSS = 0x01, // add in and out edge atomic +} (cpp.enum_strict) \ No newline at end of file diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 5e660982eda..a6b0dd93ff2 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -2804,10 +2804,10 @@ add_hosts_sentence : KW_ADD KW_HOSTS host_list { $$ = new AddHostsSentence($3); } - | KW_ADD KW_HOSTS host_list KW_INTO KW_ZONE name_label { + | KW_ADD KW_HOSTS host_list KW_INTO KW_ZONE STRING { $$ = new AddHostsIntoZoneSentence($3, $6, false); } - | KW_ADD KW_HOSTS host_list KW_INTO KW_NEW KW_ZONE name_label { + | KW_ADD KW_HOSTS host_list KW_INTO KW_NEW KW_ZONE STRING { $$ = new AddHostsIntoZoneSentence($3, $7, true); } ; @@ -2820,18 +2820,17 @@ drop_hosts_sentence merge_zone_sentence - : KW_MERGE KW_ZONE zone_name_list KW_INTO name_label { + : KW_MERGE KW_ZONE zone_name_list KW_INTO STRING { $$ = new MergeZoneSentence($3, $5); } ; drop_zone_sentence - : KW_DROP KW_ZONE name_label { + : KW_DROP KW_ZONE STRING { $$ = new DropZoneSentence($3); } ; -<<<<<<< HEAD zone_item : STRING L_PAREN host_list R_PAREN { $$ = new nebula::ZoneItem($1, $3); @@ -2852,25 +2851,20 @@ zone_item_list divide_zone_sentence : KW_DIVIDE KW_ZONE STRING KW_INTO zone_item_list { $$ = new DivideZoneSentence($3, $5); -======= -split_zone_sentence - : KW_SPLIT KW_ZONE name_label KW_INTO name_label host_list name_label host_list { - $$ = new SplitZoneSentence($3, $5, $6, $7, $8); ->>>>>>> fix } ; rename_zone_sentence - : KW_RENAME KW_ZONE name_label KW_TO name_label { + : KW_RENAME KW_ZONE STRING KW_TO STRING { $$ = new RenameZoneSentence($3, $5); } ; desc_zone_sentence - : KW_DESCRIBE KW_ZONE name_label { + : KW_DESCRIBE KW_ZONE STRING { $$ = new DescribeZoneSentence($3); } - | KW_DESC KW_ZONE name_label { + | KW_DESC KW_ZONE STRING { $$ = new DescribeZoneSentence($3); } ; @@ -3888,11 +3882,7 @@ maintain_sentence | drop_hosts_sentence { $$ = $1; } | merge_zone_sentence { $$ = $1; } | drop_zone_sentence { $$ = $1; } -<<<<<<< HEAD | divide_zone_sentence { $$ = $1; } -======= - | split_zone_sentence { $$ = $1; } ->>>>>>> fix | rename_zone_sentence { $$ = $1; } | desc_zone_sentence { $$ = $1; } | show_sentence { $$ = $1; } diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 95ccfb3d3a4..a40726b4525 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -2817,7 +2817,7 @@ TEST_F(ParserTest, Zone) { ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "ADD HOSTS 127.0.0.1:8989 INTO ZONE default_zone_127.0.0.1_8988"; + std::string query = "ADD HOSTS 127.0.0.1:8989 INTO ZONE \"default_zone_127.0.0.1_8988\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } @@ -2839,24 +2839,7 @@ TEST_F(ParserTest, Zone) { { std::string query = "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989 INTO ZONE" - " default_zone_127.0.0.1_8988"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = - "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989 INTO NEW ZONE" - " default_zone_127.0.0.1_8988"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "DESC ZONE default_zone_127.0.0.1_8988"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "DESC ZONE zone_0"; + " \"default_zone_127.0.0.1_8988\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } @@ -2878,7 +2861,7 @@ TEST_F(ParserTest, Zone) { ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "DESCRIBE ZONE default_zone_127.0.0.1_8988"; + std::string query = "DESCRIBE ZONE \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } @@ -2903,36 +2886,36 @@ TEST_F(ParserTest, Zone) { ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "MERGE ZONE zone_1,zone_2 INTO zone"; + std::string query = "MERGE ZONE \"zone_1\",\"zone_2\" INTO \"zone_1\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "MERGE ZONE zone_1,zone_2 INTO zone_1"; + std::string query = + "MERGE ZONE \"default_zone_127.0.0.1_8988\",\"default_zone_127.0.0.1_8989\"" + "INTO \"zone_1\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { std::string query = - "MERGE ZONE default_zone_127.0.0.1_8988,default_zone_127.0.0.1_8989" - "INTO zone_1"; + "MERGE ZONE \"default_zone_127.0.0.1_8988\",\"default_zone_127.0.0.1_8989\"" + "INTO \"default_zone_127.0.0.1_8989\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = - "MERGE ZONE default_zone_127.0.0.1_8988,default_zone_127.0.0.1_8989" - "INTO default_zone_127.0.0.1_8989"; + std::string query = "RENAME ZONE \"default_zone_127.0.0.1_8989\" TO \"new_name\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "RENAME ZONE default_zone_127.0.0.1_8989 TO new_name"; + std::string query = "RENAME ZONE \"old_name\" TO \"new_name\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "RENAME ZONE old_name TO new_name"; + std::string query = "RENAME ZONE \"default_zone_127.0.0.1_8989\" TO \"new_name\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 5371ad3492c..4f665611db1 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -2,6 +2,7 @@ set(storage_test_deps $ $ $ + $ $ $ $ diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index 80919a420a6..c489c751a6f 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -1,6 +1,7 @@ set(tools_test_deps $ $ + $ $ $ $ diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index 22c951d3ad9..674b8973910 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -10,6 +10,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 466b91ef62d..1e827f5dfce 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -6,6 +6,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 81de926c583..01e01040542 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -6,6 +6,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index 41f9b0a18a5..bdde2f7bcdb 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -1,6 +1,7 @@ set(perf_test_deps $ $ + $ $ $ $