diff --git a/conf/client.conf b/conf/client.conf index 9b5fe31a8e..30865c92eb 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -134,3 +134,13 @@ global.metricDummyServerStartPort=9000 # 是否关闭健康检查: true/关闭 false/不关闭 global.turnOffHealthCheck=true + +# 是否开启discard +discard.enableDiscard=true + +# discard粒度 +discard.discardGranularity=4096 + +# discard清理任务延迟时间(毫秒) +discard.discardTaskDelayMs=60000 + diff --git a/conf/cs_client.conf b/conf/cs_client.conf index 2323b83d2d..b16ff80404 100644 --- a/conf/cs_client.conf +++ b/conf/cs_client.conf @@ -142,3 +142,12 @@ global.metricDummyServerStartPort=9000 # session map文件,存储打开文件的filename到path的映射 # global.sessionMapPath=./session_map.json + +# 是否开启discard +discard.enableDiscard=false + +# discard粒度 +discard.discardGranularity=4096 + +# discard清理任务延迟时间(毫秒) +discard.discardTaskDelayMs=60000 diff --git a/conf/mds.conf b/conf/mds.conf index 07c7f673f0..bdea8d4afd 100644 --- a/conf/mds.conf +++ b/conf/mds.conf @@ -26,6 +26,8 @@ mds.segment.alloc.periodic.persistInterMs=10000 # 出错情况下的重试间隔,单位ms mds.segment.alloc.retryInterMs=1000 +mds.segment.discard.scanIntevalMs=5000 + # leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s) # 该值和etcd集群election timeout相关. diff --git a/conf/py_client.conf b/conf/py_client.conf index cdfb5e071f..12ec5832d9 100644 --- a/conf/py_client.conf +++ b/conf/py_client.conf @@ -136,3 +136,12 @@ global.metricDummyServerStartPort=10000 # session map文件,存储打开文件的filename到path的映射 # global.sessionMapPath=./session_map.json + +# 是否开启discard +discard.enableDiscard=false + +# discard粒度 +discard.discardGranularity=4096 + +# discard清理任务延迟时间(毫秒) +discard.discardTaskDelayMs=60000 diff --git a/conf/snap_client.conf b/conf/snap_client.conf index 6d5c457f6e..1df45e221e 100644 --- a/conf/snap_client.conf +++ b/conf/snap_client.conf @@ -142,3 +142,12 @@ global.metricDummyServerStartPort=9000 # session map文件,存储打开文件的filename到path的映射 # global.sessionMapPath=./session_map.json + +# 是否开启discard +discard.enableDiscard=false + +# discard粒度 +discard.discardGranularity=4096 + +# discard清理任务延迟时间(毫秒) +discard.discardTaskDelayMs=60000 diff --git a/curve-ansible/roles/generate_config/templates/client.conf.j2 b/curve-ansible/roles/generate_config/templates/client.conf.j2 index cc7eb26b1a..fa5818bd87 100644 --- a/curve-ansible/roles/generate_config/templates/client.conf.j2 +++ b/curve-ansible/roles/generate_config/templates/client.conf.j2 @@ -145,3 +145,12 @@ global.turnOffHealthCheck={{ client_turn_off_health_check }} # session map文件,存储打开文件的filename到path的映射 # global.sessionMapPath={{ client_session_map_path }} + +# 是否开启discard +discard.enableDiscard={{ client_discard_enable }} + +# discard粒度 +discard.discardGranularity={{ client_discard_granularity }} + +# discard清理任务延迟时间(毫秒) +discard.discardTaskDelayMs={{ client_discard_task_delay_ms }} diff --git a/include/client/libcurve.h b/include/client/libcurve.h index f47d7b86c0..c8370fac20 100644 --- a/include/client/libcurve.h +++ b/include/client/libcurve.h @@ -111,6 +111,7 @@ const char* ErrorNum2ErrorName(LIBCURVE_ERROR err); typedef enum LIBCURVE_OP { LIBCURVE_OP_READ, LIBCURVE_OP_WRITE, + LIBCURVE_OP_DISCARD, LIBCURVE_OP_MAX, } LIBCURVE_OP; @@ -215,6 +216,16 @@ int Read(int fd, char* buf, off_t offset, size_t length); */ int Write(int fd, const char* buf, off_t offset, size_t length); +/** + * @brief Synchronous discard operation + * @param fd file descriptor + * @param offset discard offset + * @param length discard length + * @return On success, returns the number of bytes discarded. + * On error, returns a negative value + */ +int Discard(int fd, off_t offset, size_t length); + /** * 异步模式读 * @param: fd为当前open返回的文件描述符 @@ -231,6 +242,14 @@ int AioRead(int fd, CurveAioContext* aioctx); */ int AioWrite(int fd, CurveAioContext* aioctx); +/** + * @brief Asynchronous discard operation + * @param fd file descriptor + * @param aioctx async request context + * @return 0 means success, otherwise it means failure + */ +int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 重命名文件 * @param: userinfo是用户信息 @@ -481,6 +500,14 @@ class CurveClient { virtual int AioWrite(int fd, CurveAioContext* aioctx, UserDataType dataType); + /** + * @brief Async Discard + * @param fd file descriptor + * @param aioctx async request context + * @return return error code, 0 means success + */ + virtual int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 测试使用,设置fileclient * @param client 需要设置的fileclient diff --git a/nebd/src/part2/request_executor_curve.cpp b/nebd/src/part2/request_executor_curve.cpp index 7c3a08082c..f40b9a25cb 100644 --- a/nebd/src/part2/request_executor_curve.cpp +++ b/nebd/src/part2/request_executor_curve.cpp @@ -152,13 +152,28 @@ int CurveRequestExecutor::GetInfo( return 0; } -int CurveRequestExecutor::Discard( - NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CurveRequestExecutor::Discard(NebdFileInstance* fd, + NebdServerAioContext* aioctx) { + int curveFd = GetCurveFdFromNebdFileInstance(fd); + if (curveFd < 0) { + return -1; + } - aioctx->ret = 0; - aioctx->cb(aioctx); + CurveAioCombineContext* curveCombineCtx = new CurveAioCombineContext(); + curveCombineCtx->nebdCtx = aioctx; + int ret = FromNebdCtxToCurveCtx(aioctx, &curveCombineCtx->curveCtx); + if (ret < 0) { + delete curveCombineCtx; + return -1; + } - return 0; + ret = client_->AioDiscard(curveFd, &curveCombineCtx->curveCtx); + if (ret == LIBCURVE_ERROR::OK) { + return 0; + } + + delete curveCombineCtx; + return -1; } int CurveRequestExecutor::AioRead( @@ -271,7 +286,9 @@ int CurveRequestExecutor::FromNebdOpToCurveOp(LIBAIO_OP op, LIBCURVE_OP *out) { case LIBAIO_OP::LIBAIO_OP_WRITE: *out = LIBCURVE_OP_WRITE; return 0; - + case LIBAIO_OP::LIBAIO_OP_DISCARD: + *out = LIBCURVE_OP_DISCARD; + return 0; default: return -1; } diff --git a/nebd/test/part2/mock_curve_client.h b/nebd/test/part2/mock_curve_client.h index ce12097c98..515b0fb991 100644 --- a/nebd/test/part2/mock_curve_client.h +++ b/nebd/test/part2/mock_curve_client.h @@ -45,6 +45,7 @@ class MockCurveClient : public ::curve::client::CurveClient { int(int, CurveAioContext*, curve::client::UserDataType)); MOCK_METHOD3(AioWrite, int(int, CurveAioContext*, curve::client::UserDataType)); + MOCK_METHOD2(AioDiscard, int(int, CurveAioContext*)); }; } // namespace server diff --git a/nebd/test/part2/test_request_executor_curve.cpp b/nebd/test/part2/test_request_executor_curve.cpp index 0f85914f65..072448df72 100644 --- a/nebd/test/part2/test_request_executor_curve.cpp +++ b/nebd/test/part2/test_request_executor_curve.cpp @@ -358,20 +358,58 @@ TEST_F(TestReuqestExecutorCurve, test_AioWrite) { TEST_F(TestReuqestExecutorCurve, test_Discard) { auto executor = CurveRequestExecutor::GetInstance(); + NebdServerAioContext aioctx; + aioctx.cb = NebdUnitTestCallback; std::string curveFilename("/cinder/volume-1234_cinder_"); - std::unique_ptr curveFileIns(new CurveFileInstance()); - NebdServerAioContext* aioctx = new NebdServerAioContext(); - nebd::client::DiscardResponse response; - TestReuqestExecutorCurveClosure done; - aioctx->op = LIBAIO_OP::LIBAIO_OP_DISCARD; - aioctx->cb = NebdFileServiceCallback; - aioctx->response = &response; - aioctx->done = &done; + // 1. + { + std::unique_ptr nebdFileIns(new NebdFileInstance()); + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .Times(0); + ASSERT_EQ(-1, executor.Discard(nebdFileIns.get(), &aioctx)); + } - ASSERT_EQ(0, executor.Discard(curveFileIns.get(), aioctx)); - ASSERT_TRUE(done.IsRunned()); - ASSERT_EQ(response.retcode(), nebd::client::RetCode::kOK); + // 2. + { + std::unique_ptr curveFileIns( + new CurveFileInstance()); + curveFileIns->fd = -1; + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .Times(0); + ASSERT_EQ(-1, executor.Discard(curveFileIns.get(), &aioctx)); + } + + // 3. + { + std::unique_ptr curveFileIns( + new CurveFileInstance()); + aioctx.size = 1; + aioctx.offset = 0; + aioctx.op = LIBAIO_OP::LIBAIO_OP_DISCARD; + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + ASSERT_EQ(-1, executor.Discard(curveFileIns.get(), &aioctx)); + } + + // 4 + { + std::unique_ptr curveFileIns( + new CurveFileInstance()); + aioctx.size = 1; + aioctx.offset = 0; + aioctx.op = LIBAIO_OP::LIBAIO_OP_DISCARD; + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + CurveAioContext* curveCtx; + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .WillOnce(DoAll(SaveArg<1>(&curveCtx), + Return(LIBCURVE_ERROR::OK))); + ASSERT_EQ(0, executor.Discard(curveFileIns.get(), &aioctx)); + curveCtx->cb(curveCtx); + } } TEST_F(TestReuqestExecutorCurve, test_Flush) { diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto index 53e6ffbf2b..da96e48475 100644 --- a/proto/nameserver2.proto +++ b/proto/nameserver2.proto @@ -155,6 +155,11 @@ message PageFileSegment { repeated PageFileChunkInfo chunks = 5; } +message DiscardSegmentInfo { + required FileInfo fileInfo = 1; + required PageFileSegment pageFileSegment = 2; +} + message CreateFileRequest { required string fileName = 1; required FileType fileType = 3; @@ -211,6 +216,18 @@ message GetOrAllocateSegmentResponse { optional PageFileSegment pageFileSegment = 2; } +message DeAllocateSegmentRequest { + required string fileName = 1; + required string owner = 2; + required uint64 offset = 3; + optional string signature = 4; + required uint64 date = 5; +} + +message DeAllocateSegmentResponse { + required StatusCode statusCode = 1; +} + message RenameFileRequest { required string oldFileName = 1; required string newFileName = 2; @@ -496,6 +513,7 @@ service CurveFSService { rpc GetFileInfo(GetFileInfoRequest) returns (GetFileInfoResponse); rpc GetOrAllocateSegment(GetOrAllocateSegmentRequest) returns (GetOrAllocateSegmentResponse); + rpc DeAllocateSegment(DeAllocateSegmentRequest) returns (DeAllocateSegmentResponse); rpc RenameFile(RenameFileRequest) returns (RenameFileResponse); rpc ExtendFile(ExtendFileRequest) returns (ExtendFileResponse); rpc ChangeOwner(ChangeOwnerRequest) returns (ChangeOwnerResponse); diff --git a/src/client/client_common.h b/src/client/client_common.h index 4c312355db..e3a70b1212 100644 --- a/src/client/client_common.h +++ b/src/client/client_common.h @@ -41,12 +41,17 @@ using CopysetID = uint32_t; using LogicPoolID = uint32_t; using ChunkServerID = uint32_t; using ChunkIndex = uint32_t; +using SegmentIndex = uint32_t; using EndPoint = butil::EndPoint; using Status = butil::Status; using IOManagerID = uint64_t; +constexpr uint64_t KiB = 1024; +constexpr uint64_t MiB = 1024 * KiB; +constexpr uint64_t GiB = 1024 * MiB; + // 操作类型 enum class OpType { READ = 0, @@ -56,6 +61,7 @@ enum class OpType { CREATE_CLONE, RECOVER_CHUNK, GET_CHUNK_INFO, + DISCARD, UNKNOWN }; @@ -214,6 +220,8 @@ inline const char* OpTypeToString(OpType optype) { return "RecoverChunk"; case OpType::GET_CHUNK_INFO: return "GetChunkInfo"; + case OpType::DISCARD: + return "Discard"; case OpType::UNKNOWN: default: return "Unknown"; diff --git a/src/client/client_config.cpp b/src/client/client_config.cpp index 479cf16f93..2275ef412e 100644 --- a/src/client/client_config.cpp +++ b/src/client/client_config.cpp @@ -39,6 +39,9 @@ namespace curve { namespace client { int ClientConfig::Init(const char* configpath) { conf_.SetConfigPath(configpath); + + LOG(INFO) << "Init config from " << configpath; + if (!conf_.LoadConfig()) { LOG(ERROR) << "Load config failed, config path = " << configpath; return -1; @@ -218,6 +221,25 @@ int ClientConfig::Init(const char* configpath) { << "config no global.turnOffHealthCheck info, using default value " << fileServiceOption_.commonOpt.turnOffHealthCheck; + ret = conf_.GetBoolValue( + "discard.enableDiscard", + &fileServiceOption_.ioOpt.discardOption.enableDiscard); + LOG_IF(ERROR, ret == false) + << "config no discard.enableDiscard info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value("discard.discardGranularity", + &fileServiceOption_.ioOpt.metaCacheOpt.discardGranularity); + LOG_IF(ERROR, ret == false) + << "config no discard.discardGranularity info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value("discard.discardTaskDelayMs", + &fileServiceOption_.ioOpt.discardOption.discardTaskDelayMs); + LOG_IF(ERROR, ret == false) + << "config no discard.discardTaskDelayMs info"; + RETURN_IF_FALSE(ret); + return 0; } diff --git a/src/client/client_metric.h b/src/client/client_metric.h index ead3da7258..9893c16a50 100644 --- a/src/client/client_metric.h +++ b/src/client/client_metric.h @@ -155,6 +155,8 @@ struct MDSClientMetric { InterfaceMetric getServerList; // GetOrAllocateSegment接口统计信息 InterfaceMetric getOrAllocateSegment; + // DeAllocateSegment接口统计信息 + InterfaceMetric deAllocateSegment; // RenameFile接口统计信息 InterfaceMetric renameFile; // Extend接口统计信息 @@ -185,6 +187,7 @@ struct MDSClientMetric { refreshSession(prefix, "refreshSession"), getServerList(prefix, "getServerList"), getOrAllocateSegment(prefix, "getOrAllocateSegment"), + deAllocateSegment(prefix, "deAllocateSegment"), renameFile(prefix, "renameFile"), extendFile(prefix, "extendFile"), deleteFile(prefix, "deleteFile"), diff --git a/src/client/config_info.h b/src/client/config_info.h index 302078292b..a4da337b38 100644 --- a/src/client/config_info.h +++ b/src/client/config_info.h @@ -188,6 +188,7 @@ struct MetaCacheOption { uint32_t metacacheRPCRetryIntervalUS = 500; uint32_t metacacheGetLeaderRPCTimeOutMS = 1000; uint32_t metacacheGetLeaderBackupRequestMS = 100; + uint32_t discardGranularity = 4096; std::string metacacheGetLeaderBackupRequestLbName = "rr"; ChunkServerUnstableOption chunkserverUnstableOption; }; @@ -213,6 +214,12 @@ struct TaskThreadOption { uint32_t isolationTaskThreadPoolSize = 1; }; +// for discard +struct DiscardOption { + bool enableDiscard = false; + uint32_t discardTaskDelayMs = 1000 * 60 * 3; // 3 min +}; + /** * IOOption存储了当前io 操作所需要的所有配置信息 */ @@ -222,6 +229,7 @@ struct IOOption { MetaCacheOption metaCacheOpt; TaskThreadOption taskThreadOpt; RequestScheduleOption reqSchdulerOpt; + DiscardOption discardOption; }; /** @@ -255,6 +263,7 @@ struct FileServiceOption { LeaseOption leaseOpt; CommonConfigOpt commonOpt; MetaServerOption metaServerOpt; + DiscardOption discardOption; }; } // namespace client diff --git a/src/client/discard_task.cpp b/src/client/discard_task.cpp new file mode 100644 index 0000000000..383ca129bc --- /dev/null +++ b/src/client/discard_task.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Thu Dec 17 11:05:38 CST 2020 + * Author: wuhanqing + */ + +#include "src/client/discard_task.h" + +#include + +#include +#include +#include + +namespace curve { +namespace client { + +void DiscardTask::Run() { + const FInfo* fileInfo = metaCache_->GetFileInfo(); + uint64_t offset = + static_cast(segmentIndex_) * fileInfo->segmentsize; + FileSegment* fileSegment = metaCache_->GetFileSegment(segmentIndex_); + + fileSegment->AcquireWriteLock(); + + if (!fileSegment->IsAllDiscard()) { + LOG(WARNING) << "DiscardTask find bitmap was cleared, cancel task, " + "filename = " + << fileInfo->fullPathName << ", offset = " << offset + << ", taskid = " << timerId_; + fileSegment->ReleaseLock(); + taskManager_->OnTaskFinish(timerId_); + return; + } + + LIBCURVE_ERROR errCode = mdsClient_->DeAllocateSegment(fileInfo, offset); + if (errCode == LIBCURVE_ERROR::OK) { + fileSegment->ClearBitmap(); + metaCache_->CleanChunksInSegment(segmentIndex_); + LOG(INFO) << "DiscardTask success, filename = " + << fileInfo->fullPathName << ", offset = " << offset + << ", taskid = " << timerId_; + } else { + LOG(ERROR) << "DiscardTask failed, mds return error = " << errCode + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset << ", taskid = " << timerId_; + } + + fileSegment->ReleaseLock(); + taskManager_->OnTaskFinish(timerId_); +} + +static void RunDiscardTask(void* arg) { + DiscardTask* task = static_cast(arg); + task->Run(); +} + +void DiscardTaskManager::OnTaskFinish(bthread_timer_t timerId) { + std::lock_guard lk(mtx_); + unfinishedTasks_.erase(timerId); + cond_.notify_one(); +} + +bool DiscardTaskManager::ScheduleTask(SegmentIndex segmentIndex, + MetaCache* metaCache, + MDSClient* mdsclient, timespec abstime) { + bthread_timer_t timerId; + std::unique_ptr task( + new DiscardTask(this, segmentIndex, metaCache, mdsclient)); + + int ret = bthread_timer_add(&timerId, abstime, RunDiscardTask, task.get()); + if (ret == 0) { + task->SetId(timerId); + LOG(INFO) << "Schedule discard task success, taskid = " << task->Id(); + std::lock_guard lk(mtx_); + unfinishedTasks_.emplace(timerId, std::move(task)); + return true; + } + + LOG(ERROR) << "bthread_timer_add failed, ret = " << ret + << ", errno = " << errno << ", run task directly"; + return false; +} + +void DiscardTaskManager::Stop() { + std::unordered_set currentTasks; + + { + std::lock_guard lk(mtx_); + for (auto& kv : unfinishedTasks_) { + currentTasks.emplace(kv.first); + } + } + + for (const auto& timerId : currentTasks) { + int ret = bthread_timer_del(timerId); + + if (ret == 1) { + LOG(WARNING) << "Task is running, taskid = " << timerId; + continue; + } else if (ret == 0) { + LOG(INFO) << "Cancenl discard task success, taskid = " << timerId; + } else { + if (errno == EINVAL) { + LOG(INFO) << "Task has been completed, taskid = " << timerId; + } else { + LOG(ERROR) << "bthread_timer_del return failed, ret = " << ret + << ", errno = " << errno + << ", timerId = " << timerId; + } + } + + OnTaskFinish(timerId); + } + + std::unique_lock lk(mtx_); + while (!unfinishedTasks_.empty()) { + cond_.wait(lk); + } +} + +} // namespace client +} // namespace curve diff --git a/src/client/discard_task.h b/src/client/discard_task.h new file mode 100644 index 0000000000..ade3119692 --- /dev/null +++ b/src/client/discard_task.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Thu Dec 17 11:05:38 CST 2020 + * Author: wuhanqing + */ + +#ifndef SRC_CLIENT_DISCARD_TASK_H_ +#define SRC_CLIENT_DISCARD_TASK_H_ + +#include +#include +#include +#include + +#include + +#include "src/client/metacache.h" +#include "src/client/metacache_struct.h" + +namespace curve { +namespace client { + +class DiscardTaskManager; + +class DiscardTask { + public: + DiscardTask(DiscardTaskManager* taskManager, SegmentIndex segmentIndex, + MetaCache* metaCache, MDSClient* mdsClient) + : taskManager_(taskManager), + segmentIndex_(segmentIndex), + metaCache_(metaCache), + mdsClient_(mdsClient), + timerId_(0) {} + + void Run(); + + bthread_timer_t Id() const { + return timerId_; + } + + void SetId(bthread_timer_t id) { + timerId_ = id; + } + + private: + DiscardTaskManager* taskManager_; + SegmentIndex segmentIndex_; + MetaCache* metaCache_; + MDSClient* mdsClient_; + bthread_timer_t timerId_; + + static std::atomic taskId_; +}; + +class DiscardTaskManager { + public: + DiscardTaskManager() = default; + + void OnTaskFinish(bthread_timer_t timerId); + + bool ScheduleTask(SegmentIndex segmentIndex, MetaCache* metaCache, + MDSClient* mdsclient, timespec abstime); + + /** + * @brief Cancel all unfinished discard tasks + */ + void Stop(); + + private: + bthread::Mutex mtx_; + bthread::ConditionVariable cond_; + std::unordered_map> unfinishedTasks_; // NOLINT +}; + +} // namespace client +} // namespace curve + +#endif // SRC_CLIENT_DISCARD_TASK_H_ diff --git a/src/client/file_instance.cpp b/src/client/file_instance.cpp index 288a52b2f2..439335606d 100644 --- a/src/client/file_instance.cpp +++ b/src/client/file_instance.cpp @@ -130,6 +130,24 @@ int FileInstance::AioWrite(CurveAioContext* aioctx, UserDataType dataType) { return iomanager4file_.AioWrite(aioctx, mdsclient_, dataType); } +int FileInstance::Discard(off_t offset, size_t length) { + if (!readonly_) { + return iomanager4file_.Discard(offset, length, mdsclient_); + } + + LOG(ERROR) << "Open with read only, not support Discard"; + return -1; +} + +int FileInstance::AioDiscard(CurveAioContext* aioctx) { + if (!readonly_) { + return iomanager4file_.AioDiscard(aioctx, mdsclient_); + } + + LOG(ERROR) << "Open with read only, not support AioDiscard"; + return -1; +} + // 两种场景会造成在Open的时候返回LIBCURVE_ERROR::FILE_OCCUPIED // 1. 强制重启qemu不会调用close逻辑,然后启动的时候原来的文件sessio还没过期. // 导致再次去发起open的时候,返回被占用,这种情况可以通过load sessionmap diff --git a/src/client/file_instance.h b/src/client/file_instance.h index e729ee6a41..3426016998 100644 --- a/src/client/file_instance.h +++ b/src/client/file_instance.h @@ -109,6 +109,21 @@ class CURVE_CACHELINE_ALIGNMENT FileInstance { */ int AioWrite(CurveAioContext* aioctx, UserDataType dataType); + /** + * @param offset discard offset + * @param length discard length + * @return On success, returns the number of bytes discarded. + * On error, returns a negative value + */ + int Discard(off_t offset, size_t length); + + /** + * @brief Asynchronous discard operation + * @param aioctx async request context + * @return 0 means success, otherwise it means failure + */ + int AioDiscard(CurveAioContext* aioctx); + int Close(); void UnInitialize(); diff --git a/src/client/io_tracker.cpp b/src/client/io_tracker.cpp index 0214017540..2505191884 100644 --- a/src/client/io_tracker.cpp +++ b/src/client/io_tracker.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "src/client/splitor.h" #include "src/client/iomanager.h" @@ -30,6 +31,8 @@ #include "src/client/request_scheduler.h" #include "src/client/request_closure.h" #include "src/common/timeutility.h" +#include "src/client/metacache_struct.h" +#include "src/client/discard_task.h" namespace curve { namespace client { @@ -37,6 +40,7 @@ namespace client { using curve::chunkserver::CHUNK_OP_STATUS; std::atomic IOTracker::tracekerID_(1); +DiscardOption IOTracker::discardOption_; IOTracker::IOTracker(IOManager* iomanager, MetaCache* mc, @@ -59,6 +63,12 @@ IOTracker::IOTracker(IOManager* iomanager, opStartTimePoint_ = curve::common::TimeUtility::GetTimeofDayUs(); } +void IOTracker::ReleaseAllSegmentLocks() { + for (auto& readlock : segmentLocks_) { + readlock->ReleaseLock(); + } +} + void IOTracker::StartRead(void* buf, off_t offset, size_t length, MDSClient* mdsclient, const FInfo_t* fileInfo) { data_ = buf; @@ -176,6 +186,52 @@ void IOTracker::DoWrite(MDSClient* mdsclient, const FInfo_t* fileInfo) { } } +void IOTracker::StartDiscard(off_t offset, size_t length, MDSClient* mdsclient, + const FInfo* fileInfo, + DiscardTaskManager* taskManager) { + offset_ = offset; + length_ = length; + type_ = OpType::DISCARD; + + DoDiscard(mdsclient, fileInfo, taskManager); +} + +void IOTracker::StartAioDiscard(CurveAioContext* ctx, MDSClient* mdsclient, + const FInfo_t* fileInfo, + DiscardTaskManager* taskManager) { + aioctx_ = ctx; + offset_ = ctx->offset; + length_ = ctx->length; + type_ = OpType::DISCARD; + + DoDiscard(mdsclient, fileInfo, taskManager); +} + +void IOTracker::DoDiscard(MDSClient* mdsClient, const FInfo* fileInfo, + DiscardTaskManager* taskManager) { + int ret = Splitor::CalcDiscardSegments(this); + + if (ret == 0 && !discardSegments_.empty()) { + for (auto index : discardSegments_) { + timespec abstime = + butil::milliseconds_from_now(discardOption_.discardTaskDelayMs); + if (!taskManager->ScheduleTask(index, mc_, mdsClient, abstime)) { + LOG(ERROR) << "Schedule discard task failed"; + ret = -1; + continue; + } + + LOG(INFO) << "Schedule discard task, filename = " + << fileInfo->fullPathName + << ", segment index = " << index + << ", segment offset = " << index * GiB; + } + } + + errcode_ = ret == 0 ? LIBCURVE_ERROR::OK : LIBCURVE_ERROR::FAILED; + Done(); +} + void IOTracker::ReadSnapChunk(const ChunkIDInfo &cinfo, uint64_t seq, uint64_t offset, uint64_t len, char *buf, SnapCloneClosure* scc) { @@ -347,11 +403,19 @@ void IOTracker::HandleResponse(RequestContext* reqctx) { } } +void IOTracker::InitDiscardOption(const DiscardOption& opt) { + discardOption_ = opt; +} + int IOTracker::Wait() { return iocv_.Wait(); } void IOTracker::Done() { + if (type_ == OpType::READ || type_ == OpType::WRITE) { + ReleaseAllSegmentLocks(); + } + if (errcode_ == LIBCURVE_ERROR::OK) { uint64_t duration = TimeUtility::GetTimeofDayUs() - opStartTimePoint_; MetricHelper::UserLatencyRecord(fileMetric_, duration, type_); diff --git a/src/client/io_tracker.h b/src/client/io_tracker.h index e88b468062..9faa12fb27 100644 --- a/src/client/io_tracker.h +++ b/src/client/io_tracker.h @@ -41,12 +41,17 @@ namespace curve { namespace client { + class IOManager; +class FileSegment; +class DiscardTaskManager; // IOTracker用于跟踪一个用户IO,因为一个用户IO可能会跨chunkserver, // 因此在真正下发的时候会被拆分成多个小IO并发的向下发送,因此我们需要 // 跟踪发送的request的执行情况。 class CURVE_CACHELINE_ALIGNMENT IOTracker { + friend class Splitor; + public: /** * 构造函数 @@ -99,6 +104,13 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { */ void StartAioWrite(CurveAioContext* ctx, MDSClient* mdsclient, const FInfo_t* fileInfo); + + void StartDiscard(off_t offset, size_t length, MDSClient* mdsclient, + const FInfo_t* fileInfo, DiscardTaskManager* taskManager); + void StartAioDiscard(CurveAioContext* ctx, MDSClient* mdsclient, + const FInfo_t* fileInfo, + DiscardTaskManager* taskManager); + /** * chunk相关接口是提供给snapshot使用的,上层的snapshot和file * 接口是分开的,在IOTracker这里会将其统一,这样对下层来说不用 @@ -164,7 +176,7 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { * @return: 返回读写信息,异步IO的时候返回0或-1.0代表成功,-1代表失败 * 同步IO返回length或-1,length代表真实读写长度,-1代表读写失败 */ - int Wait(); + int Wait(); /** * 每个request都要有自己的OP类型,这里提供接口可以在io拆分的时候获取类型 @@ -206,7 +218,11 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { readDatas_[subIoIndex] = data; } + static void InitDiscardOption(const DiscardOption& opt); + private: + void ReleaseAllSegmentLocks(); + /** * 当IO返回的时候调用done,由done负责向上返回 */ @@ -249,6 +265,9 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { // perform write operation void DoWrite(MDSClient* mdsclient, const FInfo_t* fileInfo); + void DoDiscard(MDSClient* mdsclient, const FInfo_t* fileInfo, + DiscardTaskManager* taskManager); + private: // io 类型 OpType type_; @@ -287,6 +306,8 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { // 大IO被拆分成多个request,这些request放在reqlist中国保存 std::vector reqlist_; + std::vector discardSegments_; + // scheduler用来将用户线程与client自己的线程切分 // 大IO被切分之后,将切分的reqlist传给scheduler向下发送 RequestScheduler* scheduler_; @@ -310,8 +331,12 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { // 快照克隆系统异步调用回调指针 SnapCloneClosure* scc_; + std::vector segmentLocks_; + // id生成器 static std::atomic tracekerID_; + + static DiscardOption discardOption_; }; } // namespace client } // namespace curve diff --git a/src/client/iomanager4file.cpp b/src/client/iomanager4file.cpp index 63e9c89720..fae49d3da0 100644 --- a/src/client/iomanager4file.cpp +++ b/src/client/iomanager4file.cpp @@ -33,8 +33,7 @@ namespace curve { namespace client { Atomic IOManager::idRecorder_(1); -IOManager4File::IOManager4File(): scheduler_(nullptr), exit_(false) { -} +IOManager4File::IOManager4File() : scheduler_(nullptr), exit_(false) {} bool IOManager4File::Initialize(const std::string& filename, const IOOption& ioOpt, @@ -42,6 +41,8 @@ bool IOManager4File::Initialize(const std::string& filename, ioopt_ = ioOpt; mc_.Init(ioopt_.metaCacheOpt, mdsclient); + + IOTracker::InitDiscardOption(ioopt_.discardOption); Splitor::Init(ioopt_.ioSplitOpt); inflightRpcCntl_.SetMaxInflightNum( @@ -111,6 +112,8 @@ void IOManager4File::UnInitialize() { scheduler_->Fini(); } + discardTaskManager_.Stop(); + { // 这个锁保证设置exit_和delete scheduler_是原子的 // 这样保证在scheduler_被析构的时候lease线程不会使用scheduler_ @@ -209,6 +212,52 @@ int IOManager4File::AioWrite(CurveAioContext* ctx, MDSClient* mdsclient, return LIBCURVE_ERROR::OK; } +int IOManager4File::Discard(off_t offset, size_t length, MDSClient* mdsclient) { + MetricHelper::IncremUserRPSCount(fileMetric_, OpType::DISCARD); + + if (!ioopt_.discardOption.enableDiscard || + length < ioopt_.metaCacheOpt.discardGranularity) { + return length; + } + + FlightIOGuard guard(this); + + IOTracker tracker(this, &mc_, scheduler_, fileMetric_); + tracker.StartDiscard(offset, length, mdsclient, GetFileInfo(), + &discardTaskManager_); + return tracker.Wait(); +} + +int IOManager4File::AioDiscard(CurveAioContext* aioctx, MDSClient* mdsclient) { + MetricHelper::IncremUserRPSCount(fileMetric_, OpType::DISCARD); + + if (!ioopt_.discardOption.enableDiscard || + aioctx->length < ioopt_.metaCacheOpt.discardGranularity) { + aioctx->ret = aioctx->length; + aioctx->cb(aioctx); + return LIBCURVE_ERROR::OK; + } + + IOTracker* ioTracker = + new (std::nothrow) IOTracker(this, &mc_, scheduler_, fileMetric_); + + if (ioTracker == nullptr) { + aioctx->ret = -LIBCURVE_ERROR::FAILED; + aioctx->cb(aioctx); + LOG(ERROR) << "allocate tracker failed!"; + return -LIBCURVE_ERROR::FAILED; + } + + inflightCntl_.IncremInflightNum(); + auto task = [this, aioctx, mdsclient, ioTracker]() { + ioTracker->StartAioDiscard(aioctx, mdsclient, this->GetFileInfo(), + &discardTaskManager_); + }; + + taskPool_.Enqueue(task); + return LIBCURVE_ERROR::OK; +} + void IOManager4File::UpdateFileInfo(const FInfo_t& fi) { mc_.UpdateFileInfo(fi); } diff --git a/src/client/iomanager4file.h b/src/client/iomanager4file.h index a6d544f333..e08bed18c1 100644 --- a/src/client/iomanager4file.h +++ b/src/client/iomanager4file.h @@ -40,6 +40,7 @@ #include "src/client/request_scheduler.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/task_thread_pool.h" +#include "src/client/discard_task.h" namespace curve { namespace client { @@ -64,6 +65,11 @@ class IOManager4File : public IOManager { const IOOption& ioOpt, MDSClient* mdsclient); + /** + * 析构,回收资源 + */ + void UnInitialize(); + /** * 同步模式读 * @param: buf为当前待读取的缓冲区 @@ -103,9 +109,21 @@ class IOManager4File : public IOManager { UserDataType dataType); /** - * 析构,回收资源 + * @brief Synchronous discard operation + * @param offset discard offset + * @param length discard length + * @return On success, returns the number of bytes discarded. + * On error, returns a negative value */ - void UnInitialize(); + int Discard(off_t offset, size_t length, MDSClient* mdsclient); + + /** + * @brief Asynchronous discard operation + * @param aioctx async request context + * @param mdsclient for communicate with MDS + * @return 0 means success, otherwise it means failure + */ + int AioDiscard(CurveAioContext* aioctx, MDSClient* mdsclient); /** * @brief 获取rpc发送令牌 @@ -253,6 +271,8 @@ class IOManager4File : public IOManager { // 不会有并发的情况,保证在资源被析构的时候lease续约 // 线程不会再用到这些资源. std::mutex exitMtx_; + + DiscardTaskManager discardTaskManager_; }; } // namespace client diff --git a/src/client/libcbd.h b/src/client/libcbd.h index 887a510711..196534e2fd 100644 --- a/src/client/libcbd.h +++ b/src/client/libcbd.h @@ -65,8 +65,10 @@ int cbd_ext4_open(const char* filename); int cbd_ext4_close(int fd); int cbd_ext4_pread(int fd, void* buf, off_t offset, size_t length); int cbd_ext4_pwrite(int fd, const void* buf, off_t offset, size_t length); +int cbd_ext4_pdiscard(int fd, off_t offset, size_t length); int cbd_ext4_aio_pread(int fd, CurveAioContext* context); int cbd_ext4_aio_pwrite(int fd, CurveAioContext* context); +int cbd_ext4_aio_pdiscard(int fd, CurveAioContext* context); int cbd_ext4_sync(int fd); int64_t cbd_ext4_filesize(const char* filename); @@ -76,35 +78,41 @@ int cbd_libcurve_open(const char* filename); int cbd_libcurve_close(int fd); int cbd_libcurve_pread(int fd, void* buf, off_t offset, size_t length); int cbd_libcurve_pwrite(int fd, const void* buf, off_t offset, size_t length); +int cbd_libcurve_pdiscard(int fd, off_t offset, size_t length); int cbd_libcurve_aio_pread(int fd, CurveAioContext* context); int cbd_libcurve_aio_pwrite(int fd, CurveAioContext* context); +int cbd_libcurve_aio_pdiscard(int fd, CurveAioContext* context); int cbd_libcurve_sync(int fd); int64_t cbd_libcurve_filesize(const char* filename); int cbd_libcurve_resize(const char* filename, int64_t size); #ifndef CBD_BACKEND_FAKE -#define cbd_lib_init cbd_libcurve_init -#define cbd_lib_fini cbd_libcurve_fini -#define cbd_lib_open cbd_libcurve_open -#define cbd_lib_close cbd_libcurve_close -#define cbd_lib_pread cbd_libcurve_pread -#define cbd_lib_pwrite cbd_libcurve_pwrite -#define cbd_lib_aio_pread cbd_libcurve_aio_pread -#define cbd_lib_aio_pwrite cbd_libcurve_aio_pwrite -#define cbd_lib_sync cbd_libcurve_sync -#define cbd_lib_filesize cbd_libcurve_filesize -#define cbd_lib_resize cbd_libcurve_resize +#define cbd_lib_init cbd_libcurve_init +#define cbd_lib_fini cbd_libcurve_fini +#define cbd_lib_open cbd_libcurve_open +#define cbd_lib_close cbd_libcurve_close +#define cbd_lib_pread cbd_libcurve_pread +#define cbd_lib_pwrite cbd_libcurve_pwrite +#define cbd_lib_pdiscard cbd_libcurve_pdiscard +#define cbd_lib_aio_pread cbd_libcurve_aio_pread +#define cbd_lib_aio_pwrite cbd_libcurve_aio_pwrite +#define cbd_lib_aio_pdiscard cbd_libcurve_aio_pdiscard +#define cbd_lib_sync cbd_libcurve_sync +#define cbd_lib_filesize cbd_libcurve_filesize +#define cbd_lib_resize cbd_libcurve_resize #else -#define cbd_lib_init cbd_ext4_init -#define cbd_lib_fini cbd_ext4_fini -#define cbd_lib_open cbd_ext4_open -#define cbd_lib_close cbd_ext4_close -#define cbd_lib_pread cbd_ext4_pread -#define cbd_lib_pwrite cbd_ext4_pwrite -#define cbd_lib_aio_pread cbd_ext4_aio_pread -#define cbd_lib_aio_pwrite cbd_ext4_aio_pwrite -#define cbd_lib_sync cbd_ext4_sync -#define cbd_lib_filesize cbd_ext4_filesize +#define cbd_lib_init cbd_ext4_init +#define cbd_lib_fini cbd_ext4_fini +#define cbd_lib_open cbd_ext4_open +#define cbd_lib_close cbd_ext4_close +#define cbd_lib_pread cbd_ext4_pread +#define cbd_lib_pwrite cbd_ext4_pwrite +#define cbd_lib_pdiscard cbd_ext4_pdiscard +#define cbd_lib_aio_pread cbd_ext4_aio_pread +#define cbd_lib_aio_pwrite cbd_ext4_aio_pwrite +#define cbd_lib_aio_pdiscard cbd_ext4_aio_pdiscard +#define cbd_lib_sync cbd_ext4_sync +#define cbd_lib_filesize cbd_ext4_filesize #endif #ifdef __cplusplus diff --git a/src/client/libcbd_ext4.cpp b/src/client/libcbd_ext4.cpp index e4e97989fe..458dd49812 100644 --- a/src/client/libcbd_ext4.cpp +++ b/src/client/libcbd_ext4.cpp @@ -75,6 +75,10 @@ int cbd_ext4_pwrite(int fd, const void* buf, off_t offset, size_t length) { return pwrite(fd, buf, length, offset); } +int cbd_ext4_pdiscard(int fd, off_t offset, size_t length) { + return length; +} + void cbd_ext4_aio_callback(union sigval sigev_value) { CurveAioContext* context = (CurveAioContext *)sigev_value.sival_ptr; //NOLINT context->cb(context); @@ -120,6 +124,12 @@ int cbd_ext4_aio_pwrite(int fd, CurveAioContext* context) { return aio_write(cb); } +int cbd_ext4_aio_pdiscard(int fd, CurveAioContext* aioctx) { + aioctx->ret = aioctx->length; + aioctx->cb(aioctx); + return 0; +} + int cbd_ext4_sync(int fd) { return fsync(fd); } diff --git a/src/client/libcbd_libcurve.cpp b/src/client/libcbd_libcurve.cpp index 70771d5b2c..d7e5598c70 100644 --- a/src/client/libcbd_libcurve.cpp +++ b/src/client/libcbd_libcurve.cpp @@ -73,6 +73,10 @@ int cbd_libcurve_pwrite(int fd, const void* buf, off_t offset, size_t length) { return Write(fd, reinterpret_cast(buf), offset, length); } +int cbd_libcurve_pdiscard(int fd, off_t offset, size_t length) { + return Discard(fd, offset, length); +} + int cbd_libcurve_aio_pread(int fd, CurveAioContext* context) { return AioRead(fd, context); } @@ -81,6 +85,10 @@ int cbd_libcurve_aio_pwrite(int fd, CurveAioContext* context) { return AioWrite(fd, context); } +int cbd_libcurve_aio_pdiscard(int fd, CurveAioContext* context) { + return AioDiscard(fd, context); +} + int cbd_libcurve_sync(int fd) { // Ignored as it always sync writes to chunkserver currently return 0; diff --git a/src/client/libcurve_client.cpp b/src/client/libcurve_client.cpp index 3c0f6cf234..c1b029ec39 100644 --- a/src/client/libcurve_client.cpp +++ b/src/client/libcurve_client.cpp @@ -118,6 +118,10 @@ int CurveClient::AioWrite(int fd, CurveAioContext* aioctx, return fileClient_->AioWrite(fd, aioctx, dataType); } +int CurveClient::AioDiscard(int fd, CurveAioContext* aioctx) { + return fileClient_->AioDiscard(fd, aioctx); +} + void CurveClient::SetFileClient(FileClient* client) { delete fileClient_; fileClient_ = client; diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp index e28465437a..b2663418ba 100644 --- a/src/client/libcurve_file.cpp +++ b/src/client/libcurve_file.cpp @@ -316,6 +316,23 @@ int FileClient::Write(int fd, const char* buf, off_t offset, size_t len) { return fileserviceMap_[fd]->Write(buf, offset, len); } +int FileClient::Discard(int fd, off_t offset, size_t length) { + if (CheckAligned(offset, length) == false) { + LOG(ERROR) << "request not aligned to 4096, offset = " << offset + << ", length = " << length; + return -LIBCURVE_ERROR::NOT_ALIGNED; + } + + ReadLockGuard lk(rwlock_); + auto iter = fileserviceMap_.find(fd); + if (CURVE_UNLIKELY(iter == fileserviceMap_.end())) { + LOG(ERROR) << "invalid fd, fd = " << fd; + return -LIBCURVE_ERROR::BAD_FD; + } + + return iter->second->Discard(offset, length); +} + int FileClient::AioRead(int fd, CurveAioContext* aioctx, UserDataType dataType) { // 长度为0,直接返回,不做任何操作 @@ -362,6 +379,24 @@ int FileClient::AioWrite(int fd, CurveAioContext* aioctx, return ret; } +int FileClient::AioDiscard(int fd, CurveAioContext* aioctx) { + if (CheckAligned(aioctx->offset, aioctx->length) == false) { + LOG(ERROR) << "request not aligned to 4096, offset = " << aioctx->offset + << ", length = " << aioctx->length; + return -LIBCURVE_ERROR::NOT_ALIGNED; + } + + int ret = -LIBCURVE_ERROR::FAILED; + ReadLockGuard lk(rwlock_); + auto iter = fileserviceMap_.find(fd); + if (CURVE_UNLIKELY(iter == fileserviceMap_.end())) { + LOG(ERROR) << "invalid fd"; + return -LIBCURVE_ERROR::BAD_FD; + } else { + return iter->second->AioDiscard(aioctx); + } +} + int FileClient::Rename(const UserInfo_t& userinfo, const std::string& oldpath, const std::string& newpath) { LIBCURVE_ERROR ret; @@ -696,6 +731,15 @@ int Write(int fd, const char* buf, off_t offset, size_t length) { return globalclient->Write(fd, buf, offset, length); } +int Discard(int fd, off_t offset, size_t length) { + if (globalclient == nullptr) { + LOG(ERROR) << "Not inited!"; + return -LIBCURVE_ERROR::FAILED; + } + + return globalclient->Discard(fd, offset, length); +} + int AioRead(int fd, CurveAioContext* aioctx) { if (globalclient == nullptr) { LOG(ERROR) << "not inited!"; @@ -721,6 +765,15 @@ int AioWrite(int fd, CurveAioContext* aioctx) { return globalclient->AioWrite(fd, aioctx); } +int AioDiscard(int fd, CurveAioContext* aioctx) { + if (globalclient == nullptr) { + LOG(ERROR) << "Not inited!"; + return -LIBCURVE_ERROR::FAILED; + } + + return globalclient->AioDiscard(fd, aioctx); +} + int Create(const char* filename, const C_UserInfo_t* userinfo, size_t size) { if (globalclient == nullptr) { LOG(ERROR) << "not inited!"; diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h index 6637d9ff9d..ebb2ea60ee 100644 --- a/src/client/libcurve_file.h +++ b/src/client/libcurve_file.h @@ -131,6 +131,16 @@ class FileClient { */ virtual int Write(int fd, const char* buf, off_t offset, size_t length); + /** + * @brief Synchronous discard operation + * @param fd file descriptor + * @param offset discard offset + * @param length discard length + * @return On success, returns the number of bytes discarded. + * On error, returns a negative value + */ + virtual int Discard(int fd, off_t offset, size_t length); + /** * 异步模式读 * @param: fd为当前open返回的文件描述符 @@ -151,6 +161,14 @@ class FileClient { virtual int AioWrite(int fd, CurveAioContext* aioctx, UserDataType dataType = UserDataType::RawBuffer); + /** + * @brief Asynchronous discard operation + * @param fd file descriptor + * @param aioctx async request context + * @return 0 means success, otherwise it means failure + */ + virtual int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 重命名文件 * @param: userinfo是用户信息 @@ -272,7 +290,7 @@ class FileClient { private: bool StartDummyServer(); - bool CheckAligned(off_t offset, size_t length) { + bool CheckAligned(off_t offset, size_t length) const { return (offset % IO_ALIGNED_BLOCK_SIZE == 0) && (length % IO_ALIGNED_BLOCK_SIZE == 0); } diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp index 373f2fa523..c136591cf1 100644 --- a/src/client/mds_client.cpp +++ b/src/client/mds_client.cpp @@ -47,6 +47,7 @@ using curve::mds::CreateFileResponse; using curve::mds::DeleteFileResponse; using curve::mds::GetFileInfoResponse; using curve::mds::GetOrAllocateSegmentResponse; +using curve::mds::DeAllocateSegmentResponse; using curve::mds::RenameFileResponse; using curve::mds::ExtendFileResponse; using curve::mds::ChangeOwnerResponse; @@ -922,6 +923,42 @@ LIBCURVE_ERROR MDSClient::GetOrAllocateSegment(bool allocate, return rpcExcutor.DoRPCTask(task, IOPathMaxRetryMS); } +LIBCURVE_ERROR MDSClient::DeAllocateSegment(const FInfo* fileInfo, + uint64_t offset) { + auto task = RPCTaskDefine { + DeAllocateSegmentResponse response; + mdsClientMetric_.deAllocateSegment.qps.count << 1; + LatencyGuard lg(&mdsClientMetric_.deAllocateSegment.latency); + mdsClientBase_.DeAllocateSegment(fileInfo, offset, &response, cntl, + channel); + + if (cntl->Failed()) { + mdsClientMetric_.deAllocateSegment.eps.count << 1; + LOG(WARNING) << "DeAllocateSegment failed, error = " + << cntl->ErrorText() + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset; + return -cntl->ErrorCode(); + } + + auto statusCode = response.statuscode(); + if (statusCode == StatusCode::kOK || + statusCode == StatusCode::kSegmentNotAllocated) { + return LIBCURVE_ERROR::OK; + } else { + LOG(WARNING) << "DeAllocateSegment mds return failed, error = " + << mds::StatusCode_Name(statusCode) + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset; + LIBCURVE_ERROR errCode; + MDSStatusCode2LibcurveError(statusCode, &errCode); + return errCode; + } + }; + + return rpcExcutor.DoRPCTask(task, metaServerOpt_.mdsMaxRetryMS); +} + LIBCURVE_ERROR MDSClient::RenameFile(const UserInfo_t& userinfo, const std::string& origin, const std::string& destination, diff --git a/src/client/mds_client.h b/src/client/mds_client.h index 4916c5aa0f..9698184486 100644 --- a/src/client/mds_client.h +++ b/src/client/mds_client.h @@ -56,7 +56,7 @@ struct LeaseRefreshResult; class MDSClient { public: MDSClient() = default; - ~MDSClient() = default; + virtual ~MDSClient() = default; using RPCFunc = std::function; @@ -127,6 +127,10 @@ class MDSClient { uint64_t offset, const FInfo_t* fi, SegmentInfo* segInfo); + + virtual LIBCURVE_ERROR DeAllocateSegment(const FInfo* fileInfo, + uint64_t offset); + /** * 获取文件信息,fi是出参 * @param: filename是文件名 diff --git a/src/client/mds_client_base.cpp b/src/client/mds_client_base.cpp index 35ff229c95..f0af627a01 100644 --- a/src/client/mds_client_base.cpp +++ b/src/client/mds_client_base.cpp @@ -364,6 +364,25 @@ void MDSClientBase::GetOrAllocateSegment(bool allocate, stub.GetOrAllocateSegment(cntl, &request, response, NULL); } +void MDSClientBase::DeAllocateSegment(const FInfo* fileInfo, + uint64_t segmentOffset, + DeAllocateSegmentResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel) { + DeAllocateSegmentRequest request; + request.set_filename(fileInfo->fullPathName); + request.set_offset(segmentOffset); + + FillUserInfo(&request, fileInfo->userinfo); + + LOG(INFO) << "DeAllocateSegent: filename = " << fileInfo->fullPathName + << ", offset = " << segmentOffset + << ", logid = " << cntl->log_id(); + + curve::mds::CurveFSService_Stub stub(channel); + stub.DeAllocateSegment(cntl, &request, response, nullptr); +} + void MDSClientBase::RenameFile(const UserInfo_t& userinfo, const std::string& origin, const std::string& destination, diff --git a/src/client/mds_client_base.h b/src/client/mds_client_base.h index 4703098662..e05241c572 100644 --- a/src/client/mds_client_base.h +++ b/src/client/mds_client_base.h @@ -76,6 +76,8 @@ using curve::mds::SetCloneFileStatusRequest; using curve::mds::SetCloneFileStatusResponse; using curve::mds::GetOrAllocateSegmentRequest; using curve::mds::GetOrAllocateSegmentResponse; +using curve::mds::DeAllocateSegmentRequest; +using curve::mds::DeAllocateSegmentResponse; using curve::mds::CheckSnapShotStatusRequest; using curve::mds::CheckSnapShotStatusResponse; using curve::mds::ListSnapShotFileInfoRequest; @@ -330,6 +332,11 @@ class MDSClientBase { GetOrAllocateSegmentResponse* response, brpc::Controller* cntl, brpc::Channel* channel); + + void DeAllocateSegment(const FInfo* fileInfo, uint64_t segmentOffset, + DeAllocateSegmentResponse* response, + brpc::Controller* cntl, brpc::Channel* channel); + /** * @brief 重名文件 * @param:userinfo 用户信息 diff --git a/src/client/metacache.cpp b/src/client/metacache.cpp index 4cfcd627e0..0c0c1aa2b1 100644 --- a/src/client/metacache.cpp +++ b/src/client/metacache.cpp @@ -67,6 +67,12 @@ MetaCacheErrorType MetaCache::GetChunkInfoByIndex(ChunkIndex chunkidx, return MetaCacheErrorType::CHUNKINFO_NOT_FOUND; } +void MetaCache::UpdateChunkInfoByIndex(ChunkIndex cindex, + const ChunkIDInfo& cinfo) { + WriteLockGuard wrlk(rwlock4ChunkInfo_); + chunkindex2idMap_[cindex] = cinfo; +} + bool MetaCache::IsLeaderMayChange(LogicPoolID logicPoolId, CopysetID copysetId) { rwlock4ChunkInfo_.RDLock(); @@ -262,12 +268,6 @@ int MetaCache::UpdateLeader(LogicPoolID logicPoolId, return iter->second.UpdateLeaderInfo(csAddr); } -void MetaCache::UpdateChunkInfoByIndex(ChunkIndex cindex, - const ChunkIDInfo& cinfo) { - WriteLockGuard wrlk(rwlock4ChunkInfo_); - chunkindex2idMap_[cindex] = cinfo; -} - void MetaCache::UpdateCopysetInfo(LogicPoolID logicPoolid, CopysetID copysetid, const CopysetInfo& csinfo) { const auto key = CalcLogicPoolCopysetID(logicPoolid, copysetid); @@ -415,5 +415,44 @@ CopysetInfo MetaCache::GetCopysetinfo(LogicPoolID lpid, CopysetID csid) { return CopysetInfo(); } +FileSegment* MetaCache::GetFileSegment(SegmentIndex segmentIndex) { + { + ReadLockGuard lk(rwlock4Segments_); + auto iter = segments_.find(segmentIndex); + if (iter != segments_.end()) { + return &iter->second; + } + } + + { + WriteLockGuard lk(rwlock4Segments_); + auto ret = segments_.emplace( + std::piecewise_construct, + std::forward_as_tuple(segmentIndex), + std::forward_as_tuple(segmentIndex, + fileInfo_.segmentsize, + metacacheopt_.discardGranularity)); + + return &(ret.first->second); + } + + ReadLockGuard lk(rwlock4Segments_); + return &segments_.at(segmentIndex); +} + +void MetaCache::CleanChunksInSegment(SegmentIndex segmentIndex) { + WriteLockGuard lk(rwlock4chunkInfoMap_); + ChunkIndex beginChunkIndex = static_cast(segmentIndex) * + fileInfo_.segmentsize / fileInfo_.chunksize; + ChunkIndex endChunkIndex = static_cast(segmentIndex + 1) * + fileInfo_.segmentsize / fileInfo_.chunksize; + + auto currentIndex = beginChunkIndex; + while (currentIndex < endChunkIndex) { + chunkindex2idMap_.erase(currentIndex); + ++currentIndex; + } +} + } // namespace client } // namespace curve diff --git a/src/client/metacache.h b/src/client/metacache.h index a24f3cee11..10d2dbf6db 100644 --- a/src/client/metacache.h +++ b/src/client/metacache.h @@ -81,6 +81,14 @@ class MetaCache { virtual MetaCacheErrorType GetChunkInfoByIndex(ChunkIndex chunkidx, ChunkIDInfo_t* chunkinfo); + /** + * 通过chunk index更新chunkid信息 + * @param: index为待更新的chunk index + * @param: chunkinfo为需要更新的info信息 + */ + virtual void UpdateChunkInfoByIndex(ChunkIndex cindex, + const ChunkIDInfo& chunkinfo); + /** * sender发送数据的时候需要知道对应的leader然后发送给对应的chunkserver * 如果get不到的时候,外围设置refresh为true,然后向chunkserver端拉取最新的 @@ -121,13 +129,8 @@ class MetaCache { virtual void UpdateCopysetInfo(LogicPoolID logicPoolId, CopysetID copysetId, const CopysetInfo& csinfo); - /** - * 通过chunk index更新chunkid信息 - * @param: index为待更新的chunk index - * @param: chunkinfo为需要更新的info信息 - */ - virtual void UpdateChunkInfoByIndex(ChunkIndex cindex, - const ChunkIDInfo& chunkinfo); + + /** * 通过chunk id更新chunkid信息 * @param: cid为chunkid @@ -237,6 +240,16 @@ class MetaCache { return unstableHelper_; } + /** + * @brief Get file segment info about the segmentIndex + */ + FileSegment* GetFileSegment(SegmentIndex segmentIndex); + + /** + * @brief Clean chunks of this segment + */ + virtual void CleanChunksInSegment(SegmentIndex segmentIndex); + private: /** * @brief 从mds更新copyset复制组信息 @@ -276,6 +289,9 @@ class MetaCache { // chunkindex到chunkidinfo的映射表 CURVE_CACHELINE_ALIGNMENT ChunkIndexInfoMap chunkindex2idMap_; + CURVE_CACHELINE_ALIGNMENT RWLock rwlock4Segments_; + CURVE_CACHELINE_ALIGNMENT std::unordered_map segments_; // NOLINT + // logicalpoolid和copysetid到copysetinfo的映射表 CURVE_CACHELINE_ALIGNMENT CopysetInfoMap lpcsid2CopsetInfoMap_; diff --git a/src/client/metacache_struct.h b/src/client/metacache_struct.h index af841cf688..cfd0ec391d 100644 --- a/src/client/metacache_struct.h +++ b/src/client/metacache_struct.h @@ -31,11 +31,17 @@ #include "include/curve_compiler_specific.h" #include "src/client/client_common.h" #include "src/common/concurrent/spinlock.h" +#include "src/common/concurrent/rw_lock.h" +#include "src/common/bitmap.h" namespace curve { namespace client { using curve::common::SpinLock; +using curve::common::ReadLockGuard; +using curve::common::WriteLockGuard; +using curve::common::Bitmap; +using curve::common::BthreadRWLock; // copyset内的chunkserver节点的基本信息 // 包含当前chunkserver的id信息,以及chunkserver的地址信息 @@ -264,6 +270,140 @@ inline bool operator==(const CopysetIDInfo& cpidinfo1, return cpidinfo1.cpid == cpidinfo2.cpid && cpidinfo1.lpid == cpidinfo2.lpid; } +class FileSegment { + public: + FileSegment(SegmentIndex segmentIndex, uint32_t segmentSize, + uint32_t discardGranularity) + : segmentIndex_(segmentIndex), + segmentSize_(segmentSize), + discardGranularity_(discardGranularity), + rwlock_(), + discardBitmap_(segmentSize_ / discardGranularity_), + chunks_() {} + + /** + * @brief Test if all bit was discarded + * @return Return true if if all bits are set, otherwise return false + */ + bool IsAllDiscard() { + return discardBitmap_.NextClearBit(0) == curve::common::Bitmap::NO_POS; + } + + void AcquireReadLock() { + rwlock_.RDLock(); + } + + void AcquireWriteLock() { + rwlock_.WRLock(); + } + + void ReleaseLock() { + rwlock_.Unlock(); + } + + /** + * @brief Get internal bitmap for unit-test + * @return Internal bitmap + */ + Bitmap& GetBitmap() { + return discardBitmap_; + } + + void SetDiscard(const uint64_t offset, const uint32_t length); + void ClearDiscard(const uint64_t offset, const uint32_t length); + + void ClearBitmap() { + discardBitmap_.Clear(); + } + + private: + const SegmentIndex segmentIndex_; + const uint32_t segmentSize_; + const uint32_t discardGranularity_; + BthreadRWLock rwlock_; + Bitmap discardBitmap_; + std::unordered_map chunks_; +}; + +inline void FileSegment::SetDiscard(uint64_t offset, uint32_t length) { + if (length < discardGranularity_) { + return; + } + + if (offset == 0 && length == segmentSize_) { + return discardBitmap_.Set(); + } + + auto res = std::div(static_cast(offset), + static_cast(discardGranularity_)); + uint32_t startIndex = res.quot; + if (res.rem != 0) { + ++startIndex; + } + + uint32_t endIndex = (offset + length) / discardGranularity_ - 1; + + return discardBitmap_.Set(startIndex, endIndex); +} + +inline void FileSegment::ClearDiscard(uint64_t offset, uint32_t length) { + if (offset == 0 && length == segmentSize_) { + return discardBitmap_.Clear(); + } + + uint32_t startIndex = offset / discardGranularity_; + auto res = std::div(static_cast(offset + length), + static_cast(discardGranularity_)); + + uint32_t endIndex = res.quot; + if (res.rem == 0 && endIndex != 0) { + --endIndex; + } + + return discardBitmap_.Clear(startIndex, endIndex); +} + +enum class FileSegmentLockType { + Read, + Write +}; + +template +class FileSegmentLockGuard { + public: + explicit FileSegmentLockGuard(FileSegment* segment) : segment_(segment) { + Lock(); + } + + FileSegmentLockGuard(const FileSegmentLockGuard&) = delete; + FileSegmentLockGuard& operator=(const FileSegmentLockGuard&) = delete; + + ~FileSegmentLockGuard() { + UnLock(); + } + + void Lock() { + if (type == FileSegmentLockType::Read) { + segment_->AcquireReadLock(); + } else { + segment_->AcquireWriteLock(); + } + } + + void UnLock() { + segment_->ReleaseLock(); + } + + private: + FileSegment* segment_; +}; + +using FileSegmentReadLockGuard = + FileSegmentLockGuard; + +using FileSegmentWriteLockGuard = + FileSegmentLockGuard; + } // namespace client } // namespace curve diff --git a/src/client/splitor.cpp b/src/client/splitor.cpp index 2e7d0dc16f..651f18f15b 100644 --- a/src/client/splitor.cpp +++ b/src/client/splitor.cpp @@ -109,6 +109,45 @@ int Splitor::IO2ChunkRequests(IOTracker* iotracker, MetaCache* metaCache, return 0; } +int Splitor::CalcDiscardSegments(IOTracker* iotracker) { + if (iotracker == nullptr) { + return -1; + } + + MetaCache* metaCache = iotracker->mc_; + uint64_t offset = iotracker->offset_; + uint32_t length = iotracker->length_; + const FInfo* fileInfo = metaCache->GetFileInfo(); + const uint64_t segmentSize = fileInfo->segmentsize; + + SegmentIndex beginIndex = offset / segmentSize; + SegmentIndex endIndex = (offset + length - 1) / segmentSize; + uint64_t currentOffset = offset; + + while (beginIndex <= endIndex) { + uint64_t segmentStartOffset = beginIndex * segmentSize; + uint64_t segmentEndOffset = (beginIndex + 1) * segmentSize; + uint64_t currentDiscardLength = + std::min(segmentEndOffset, offset + length) - currentOffset; + + FileSegment* fileSegment = metaCache->GetFileSegment(beginIndex); + + // acquire segment write lock + FileSegmentLockGuard lk(fileSegment); + fileSegment->SetDiscard(currentOffset - segmentStartOffset, + currentDiscardLength); + + if (fileSegment->IsAllDiscard()) { + iotracker->discardSegments_.push_back(beginIndex); + } + + ++beginIndex; + currentOffset += currentDiscardLength; + } + + return 0; +} + // this offset is begin by chunk int Splitor::SingleChunkIO2ChunkRequests( IOTracker* iotracker, MetaCache* metaCache, @@ -171,10 +210,23 @@ int Splitor::SingleChunkIO2ChunkRequests( bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, std::vector* targetlist, butil::IOBuf* data, off_t off, size_t len, - MDSClient* mdsclient, const FInfo_t* fileinfo, + MDSClient* mdsclient, const FInfo_t* fileInfo, ChunkIndex chunkidx) { const auto maxSplitSizeBytes = 1024 * iosplitopt_.fileIOSplitMaxSizeKB; + lldiv_t res = std::div( + static_cast(chunkidx) * fileInfo->chunksize, // NOLINT + static_cast(fileInfo->segmentsize)); // NOLINT + + SegmentIndex segmentIndex = res.quot; + uint64_t clearStartOffset = res.rem + off; + + FileSegment* fileSegment = metaCache->GetFileSegment(segmentIndex); + FileSegmentLockGuard lk(fileSegment); + + // clear discard bitmap + fileSegment->ClearDiscard(clearStartOffset, len); + ChunkIDInfo chunkIdInfo; MetaCacheErrorType errCode = metaCache->GetChunkInfoByIndex(chunkidx, &chunkIdInfo); @@ -182,8 +234,8 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, if (errCode == MetaCacheErrorType::CHUNKINFO_NOT_FOUND) { if (false == GetOrAllocateSegment( true, - static_cast(chunkidx) * fileinfo->chunksize, - mdsclient, metaCache, fileinfo)) { + static_cast(chunkidx) * fileInfo->chunksize, + mdsclient, metaCache, fileInfo)) { return false; } @@ -203,7 +255,7 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, std::vector templist; ret = SingleChunkIO2ChunkRequests(iotracker, metaCache, &templist, chunkIdInfo, data, off, len, - fileinfo->seqnum); + fileInfo->seqnum); for (auto& ctx : templist) { ctx->appliedindex_ = appliedindex_; @@ -214,6 +266,12 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, targetlist->insert(targetlist->end(), templist.begin(), templist.end()); + if (ret == 0) { + // acquire filesegment read lock + fileSegment->AcquireReadLock(); + iotracker->segmentLocks_.emplace_back(fileSegment); + } + return ret == 0; } diff --git a/src/client/splitor.h b/src/client/splitor.h index 6975250b23..0933c957c1 100644 --- a/src/client/splitor.h +++ b/src/client/splitor.h @@ -60,6 +60,9 @@ class Splitor { size_t length, MDSClient* mdsclient, const FInfo_t* fi); + + static int CalcDiscardSegments(IOTracker* iotracker); + /** * 对单ChunkIO进行细粒度拆分 * @param: iotracker大IO上下文信息 diff --git a/src/common/encode.h b/src/common/encode.h index ab23b24ad2..5777e39d4f 100644 --- a/src/common/encode.h +++ b/src/common/encode.h @@ -27,6 +27,8 @@ namespace curve { namespace common { + +// NOTE: value passed to this function will convert to `uint64_t' static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[0] = (value >> 56) & 0xff; buf[1] = (value >> 48) & 0xff; @@ -37,6 +39,11 @@ static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[6] = (value >> 8) & 0xff; buf[7] = value & 0xff; } + +// https://stackoverflow.com/questions/12877546/how-do-i-avoid-implicit-conversions-on-non-constructing-functions +// template +// static inline void EncodeBigEndian(char*, T) = delete; + } // namespace common } // namespace curve diff --git a/src/common/namespace_define.h b/src/common/namespace_define.h index 3f142d527f..442b7e36c4 100644 --- a/src/common/namespace_define.h +++ b/src/common/namespace_define.h @@ -59,10 +59,14 @@ const char SNAPINFOKEYEND[] = "12"; const char CLONEINFOKEYPREFIX[] = "12"; const char CLONEINFOKEYEND[] = "13"; +const char DISCARDSEGMENTKEYPREFIX[] = "13"; +const char DISCARDSEGMENTKEYEND[] = "14"; + // TODO(hzsunjianliang): if use single prefix for snapshot file? const int COMMON_PREFIX_LENGTH = 2; const int LEADER_PREFIX_LENGTH = 8; const int SEGMENTKEYLEN = 18; +const int DISCARDSEGMENTKEYLEN = 26; } // namespace common } // namespace curve diff --git a/src/kvstorageclient/etcd_client.cpp b/src/kvstorageclient/etcd_client.cpp index 96147fcde9..1d80cc96b5 100644 --- a/src/kvstorageclient/etcd_client.cpp +++ b/src/kvstorageclient/etcd_client.cpp @@ -96,8 +96,26 @@ int EtcdClientImp::Get(const std::string &key, std::string *out) { return errCode; } -int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, - std::vector *out) { +int EtcdClientImp::List(const std::string& startKey, const std::string& endKey, + std::vector* out) { + assert(out != nullptr); + out->clear(); + + std::vector> kvs; + int errCode = List(startKey, endKey, &kvs); + if (errCode != EtcdErrCode::EtcdOK) { + return errCode; + } + + for (const auto& kv : kvs) { + out->push_back(kv.second); + } + + return errCode; +} + +int EtcdClientImp::List(const std::string& startKey, const std::string& endKey, + std::vector>* out) { assert(out != nullptr); out->clear(); @@ -113,8 +131,9 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, needRetry = NeedRetry(errCode); if (res.r0 != EtcdErrCode::EtcdOK) { LOG(WARNING) << "list file of [start:" << startKey - << ", end:" << endKey << "] err: " << res.r0 - << ", retry: " << retry << ", needRetry: " << needRetry; + << ", end:" << endKey << "] err: " << res.r0 + << ", retry: " << retry + << ", needRetry: " << needRetry; } else { for (int i = 0; i < res.r2; i++) { EtcdClientGetMultiObject_return objRes = @@ -127,6 +146,7 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, } out->emplace_back( + std::string(objRes.r3, objRes.r3 + objRes.r4), std::string(objRes.r1, objRes.r1 + objRes.r2)); free(objRes.r1); free(objRes.r3); diff --git a/src/kvstorageclient/etcd_client.h b/src/kvstorageclient/etcd_client.h index 61219dc6e5..b3cd4f5200 100644 --- a/src/kvstorageclient/etcd_client.h +++ b/src/kvstorageclient/etcd_client.h @@ -26,6 +26,7 @@ #include #include #include +#include namespace curve { namespace kvstorage { @@ -78,6 +79,18 @@ class KVStorageClient { virtual int List(const std::string &startKey, const std::string &endKey, std::vector *values) = 0; + /** + * @brief List all the key and values between [startKey, endKey) + * + * @param[in] startKey + * @param[in] endKey + * @param[out] out key and values between [startKey, endKey) + * + * @return error code + */ + virtual int List(const std::string& startKey, const std::string& endKey, + std::vector>* out) = 0; + /** * @brief Delete Delete the value of the specified key * @@ -151,6 +164,9 @@ class EtcdClientImp : public KVStorageClient { int List(const std::string &startKey, const std::string &endKey, std::vector *values) override; + int List(const std::string& startKey, const std::string& endKey, + std::vector >* out) override; + int Delete(const std::string &key) override; int DeleteRewithRevision( diff --git a/src/mds/nameserver2/clean_core.cpp b/src/mds/nameserver2/clean_core.cpp index d90e891a2d..242f7a8597 100644 --- a/src/mds/nameserver2/clean_core.cpp +++ b/src/mds/nameserver2/clean_core.cpp @@ -120,25 +120,15 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile, return StatusCode::kCommonFileDeleteError; } - // delete chunks in chunkserver - LogicalPoolID logicalPoolID = segment.logicalpoolid(); - uint32_t chunkNum = segment.chunks_size(); - for (uint32_t j = 0; j != chunkNum; j++) { - SeqNum seq = commonFile.seqnum(); - int ret = copysetClient_->DeleteChunk(logicalPoolID, - segment.chunks()[j].copysetid(), - segment.chunks()[j].chunkid(), - seq); - if (ret != 0) { - LOG(ERROR) << "Clean common File Error: " - << "DeleteChunk Error" - << ", ret = " << ret - << ", inodeid = " << commonFile.id() - << ", filename = " << commonFile.filename() - << ", sequenceNum = " << seq; - progress->SetStatus(TaskStatus::FAILED); - return StatusCode::kCommonFileDeleteError; - } + int ret = DeleteChunksInSegment(segment, commonFile.seqnum()); + if (ret != 0) { + LOG(ERROR) << "Clean common File Error: " + << ", ret = " << ret + << ", inodeid = " << commonFile.id() + << ", filename = " << commonFile.filename() + << ", sequenceNum = " << commonFile.seqnum(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::kCommonFileDeleteError; } // delete segment @@ -176,5 +166,66 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile, progress->SetStatus(TaskStatus::SUCCESS); return StatusCode::kOK; } + +StatusCode CleanCore::CleanDiscardSegment( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo, TaskProgress* progress) { + const FileInfo& fileInfo = discardSegmentInfo.fileinfo(); + const PageFileSegment& segment = discardSegmentInfo.pagefilesegment(); + const LogicalPoolID logicalPoolId = segment.logicalpoolid(); + const SeqNum seq = fileInfo.seqnum(); + + // delete chunks + int ret = DeleteChunksInSegment(segment, seq); + if (ret != 0) { + LOG(ERROR) << "Clean segment failed, DeleteChunk Error, ret = " << ret + << ", filename = " << fileInfo.filename() + << ", inodeid = " << fileInfo.id() + << ", segment offset = " << segment.startoffset(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::KInternalError; + } + + // delete segment + int64_t revision; + auto storeRet = storage_->CleanDiscardSegment(cleanSegmentKey, &revision); + if (storeRet != StoreStatus::OK) { + LOG(ERROR) << "CleanDiscardSegment failed, filename = " + << fileInfo.filename() + << ", offset = " << segment.startoffset(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::KInternalError; + } + + allocStatistic_->DeAllocSpace(segment.logicalpoolid(), + segment.segmentsize(), revision); + progress->SetProgress(100); + progress->SetStatus(TaskStatus::SUCCESS); + return StatusCode::kOK; +} + +int CleanCore::DeleteChunksInSegment(const PageFileSegment& segment, + const SeqNum& seq) { + const LogicalPoolID logicalPoolId = segment.logicalpoolid(); + for (int i = 0; i < segment.chunks_size(); ++i) { + int ret = copysetClient_->DeleteChunk( + logicalPoolId, + segment.chunks()[i].copysetid(), + segment.chunks()[i].chunkid(), + seq); + + if (ret != 0) { + LOG(ERROR) << "DeleteChunk failed, ret = " << ret + << ", logicalpoolid = " << logicalPoolId + << ", copysetid = " << segment.chunks()[i].copysetid() + << ", chunkid = " << segment.chunks()[i].chunkid() + << ", seq = " << seq; + return ret; + } + } + + return 0; +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/clean_core.h b/src/mds/nameserver2/clean_core.h index 540794c1ac..0cb4f3f8ab 100644 --- a/src/mds/nameserver2/clean_core.h +++ b/src/mds/nameserver2/clean_core.h @@ -24,6 +24,7 @@ #define SRC_MDS_NAMESERVER2_CLEAN_CORE_H_ #include +#include #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/common/mds_define.h" #include "src/mds/nameserver2/task_progress.h" @@ -65,7 +66,17 @@ class CleanCore { StatusCode CleanFile(const FileInfo & commonFile, TaskProgress* progress); + /** + * @brief clean discarded segment and chunks + */ + StatusCode CleanDiscardSegment(const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo, + TaskProgress* progress); + private: + int DeleteChunksInSegment(const PageFileSegment& segment, + const SeqNum& seq); + std::shared_ptr storage_; std::shared_ptr copysetClient_; std::shared_ptr allocStatistic_; diff --git a/src/mds/nameserver2/clean_manager.cpp b/src/mds/nameserver2/clean_manager.cpp index de7447eda4..cab8458a77 100644 --- a/src/mds/nameserver2/clean_manager.cpp +++ b/src/mds/nameserver2/clean_manager.cpp @@ -57,6 +57,15 @@ bool CleanManager::SubmitDeleteCommonFileJob(const FileInfo &fileInfo) { return taskMgr_->PushTask(commonFileCleanTask); } +bool CleanManager::SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) { + auto task = std::make_shared( + cleanCore_, cleanSegmentKey, discardSegmentInfo); + task->SetTaskID(reinterpret_cast(task.get())); + return taskMgr_->PushTask(task); +} + bool CleanManager::RecoverCleanTasks(void) { // load task from store std::vector snapShotFiles; @@ -94,5 +103,44 @@ std::shared_ptr CleanManager::GetTask(TaskIDType id) { return taskMgr_->GetTask(id); } +void CleanDiscardSegmentTask::Start() { + if (running_ == false) { + LOG(INFO) << "start CleanDiscardSegmentTask"; + running_ = true; + taskThread_ = curve::common::Thread( + &CleanDiscardSegmentTask::ScanAndExecTask, this); + } +} + +void CleanDiscardSegmentTask::Stop() { + if (running_.exchange(false)) { + LOG(INFO) << "stop CleanDiscardSegmentTask..."; + sleeper_.interrupt(); + taskThread_.join(); + LOG(INFO) << "stop CleanDiscardSegmentTask success"; + } +} + +void CleanDiscardSegmentTask::ScanAndExecTask() { + std::map discardSegments; + + while (sleeper_.wait_for(std::chrono::milliseconds(scanIntervalMs_))) { + discardSegments.clear(); + auto err = storage_->ListDiscardSegment(&discardSegments); + if (err != StoreStatus::OK) { + LOG(ERROR) << "ListDiscardSegment failed"; + continue; + } + + for (const auto& kv : discardSegments) { + if (!cleanManager_->SubmitCleanDiscardSegmentJob(kv.first, + kv.second)) { + LOG(ERROR) << "SubmitCleanDiscardSegmentJob failed"; + continue; + } + } + } +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/clean_manager.h b/src/mds/nameserver2/clean_manager.h index b3940fd787..27fdaec660 100644 --- a/src/mds/nameserver2/clean_manager.h +++ b/src/mds/nameserver2/clean_manager.h @@ -23,16 +23,21 @@ #ifndef SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ #define SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ +#include #include +#include #include "proto/nameserver2.pb.h" #include "src/mds/nameserver2/clean_task_manager.h" #include "src/mds/nameserver2/clean_core.h" #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/nameserver2/async_delete_snapshot_entity.h" +#include "src/common/concurrent/concurrent.h" namespace curve { namespace mds { +class CleanDiscardSegmentTask; + class CleanManagerInterface { public: virtual ~CleanManagerInterface() {} @@ -40,6 +45,10 @@ class CleanManagerInterface { std::shared_ptr entity) = 0; virtual std::shared_ptr GetTask(TaskIDType id) = 0; virtual bool SubmitDeleteCommonFileJob(const FileInfo&) = 0; + + virtual bool SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) = 0; }; /** * CleanManager 用于异步清理 删除快照对应的数据 @@ -61,6 +70,10 @@ class CleanManager : public CleanManagerInterface { bool SubmitDeleteCommonFileJob(const FileInfo&fileInfo) override; + bool SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) override; + bool RecoverCleanTasks(void); std::shared_ptr GetTask(TaskIDType id) override; @@ -71,6 +84,35 @@ class CleanManager : public CleanManagerInterface { std::shared_ptr taskMgr_; }; +class CleanDiscardSegmentTask { + public: + CleanDiscardSegmentTask(std::shared_ptr cleanManager, + std::shared_ptr storage, + uint32_t scanIntervalMs) + : cleanManager_(cleanManager), + storage_(storage), + scanIntervalMs_(scanIntervalMs), + sleeper_(), + running_(false), + taskThread_() {} + + void Start(); + + void Stop(); + + private: + void ScanAndExecTask(); + + private: + std::shared_ptr cleanManager_; + std::shared_ptr storage_; + uint32_t scanIntervalMs_; + curve::common::InterruptibleSleeper sleeper_; + curve::common::Atomic running_; + curve::common::Thread taskThread_; +}; + } // namespace mds } // namespace curve + #endif // SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ diff --git a/src/mds/nameserver2/clean_task.h b/src/mds/nameserver2/clean_task.h index 415b4b5bbf..2f93397bcc 100644 --- a/src/mds/nameserver2/clean_task.h +++ b/src/mds/nameserver2/clean_task.h @@ -25,6 +25,7 @@ #include #include //NOLINT +#include #include //NOLINT #include //NOLINT #include "proto/nameserver2.pb.h" @@ -138,6 +139,28 @@ class CommonFileCleanTask: public Task { FileInfo fileInfo_; }; +class SegmentCleanTask : public Task { + public: + SegmentCleanTask(std::shared_ptr cleanCore, + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) + : Task(), + cleanCore_(cleanCore), + cleanSegmentKey_(cleanSegmentKey), + discardSegmentInfo_(discardSegmentInfo) {} + + void Run() override { + cleanCore_->CleanDiscardSegment(cleanSegmentKey_, discardSegmentInfo_, + GetMutableTaskProgress()); + return; + } + + private: + std::shared_ptr cleanCore_; + std::string cleanSegmentKey_; + DiscardSegmentInfo discardSegmentInfo_; +}; + } // namespace mds } // namespace curve #endif // SRC_MDS_NAMESERVER2_CLEAN_TASK_H_ diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp index 4443a4fe58..057110ac76 100644 --- a/src/mds/nameserver2/curvefs.cpp +++ b/src/mds/nameserver2/curvefs.cpp @@ -39,6 +39,23 @@ using curve::mds::topology::LogicalPool; namespace curve { namespace mds { + +inline bool CurveFS::CheckSegmentOffset(const FileInfo& fileInfo, + uint64_t offset) const { + if (offset % fileInfo.segmentsize() != 0) { + LOG(WARNING) << "offset not align with segment, offset = " << offset + << ", file segmentsize = " << fileInfo.segmentsize(); + return false; + } + + if (offset + fileInfo.segmentsize() > fileInfo.length()) { + LOG(WARNING) << "bigger than file length"; + return false; + } + + return true; +} + bool CurveFS::InitRecycleBinDir() { FileInfo recyclebinFileInfo; @@ -1022,6 +1039,72 @@ StatusCode CurveFS::GetOrAllocateSegment(const std::string & filename, } } +StatusCode CurveFS::DeAllocateSegment(const std::string& fileName, + uint64_t offset) { + FileInfo fileInfo; + auto ret = GetFileInfo(fileName, &fileInfo); + if (ret != StatusCode::kOK) { + LOG(ERROR) << "get source file error, errCode = " << ret; + return ret; + } + + if (fileInfo.filetype() != FileType::INODE_PAGEFILE) { + LOG(ERROR) << "not pageFile, can't do this"; + return StatusCode::kParaError; + } + + if (CheckSegmentOffset(fileInfo, offset) == false) { + LOG(ERROR) << "DeAllocateSegment check offset failed"; + return StatusCode::kParaError; + } + + if (fileInfo.filestatus() == FileStatus::kFileBeingCloned) { + LOG(WARNING) + << "DeAllocateSegment failed, file is being cloned, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kNotSupported; + } + + PageFileSegment segment; + auto storeRet = storage_->GetSegment(fileInfo.id(), offset, &segment); + if (StoreStatus::KeyNotExist == storeRet) { + LOG(WARNING) << "DeAllocateSegment segment not exist, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kSegmentNotAllocated; + } + + if (segment.startoffset() != offset) { + LOG(ERROR) + << "DeAllocateSegment check offset failed, segment startoffset = " + << segment.startoffset() << ", request offset = " << offset + << ", filename = " << fileName; + return StatusCode::kParaError; + } + + std::vector snapShotFiles; + if (storage_->ListSnapshotFile(fileInfo.id(), fileInfo.id() + 1, + &snapShotFiles) != StoreStatus::OK) { + LOG(WARNING) << fileName << " list snapshot failed"; + return StatusCode::kStorageError; + } + + if (snapShotFiles.size() != 0) { + LOG(WARNING) << fileName + << " exist snapshot, num = " << snapShotFiles.size(); + return StatusCode::kFileUnderSnapShot; + } + + storeRet = storage_->DiscardSegment(fileInfo, segment); + if (storeRet != StoreStatus::OK) { + LOG(WARNING) << "Storage CleanSegment return error, filename = " + << fileName << ", offset = " << offset + << ", error = " << storeRet; + return StatusCode::kStorageError; + } + + return StatusCode::kOK; +} + StatusCode CurveFS::CreateSnapShotFile(const std::string &fileName, FileInfo *snapshotFileInfo) { FileInfo parentFileInfo; diff --git a/src/mds/nameserver2/curvefs.h b/src/mds/nameserver2/curvefs.h index b005f080b0..9b5384ee10 100644 --- a/src/mds/nameserver2/curvefs.h +++ b/src/mds/nameserver2/curvefs.h @@ -229,6 +229,14 @@ class CurveFS { offset_t offset, bool allocateIfNoExist, PageFileSegment *segment); + /** + * @brief deallocate file segment start at offset + * @param filename + * @param offset + * @return On success, return StatusCode::kOK + */ + StatusCode DeAllocateSegment(const std::string& filename, uint64_t offset); + /** * @brief get the root file info * @param @@ -520,6 +528,8 @@ class CurveFS { const std::string& signature, uint64_t date); + bool CheckSegmentOffset(const FileInfo& fileInfo, uint64_t offset) const; + StatusCode CheckPathOwnerInternal(const std::string &filename, const std::string &owner, const std::string &signature, diff --git a/src/mds/nameserver2/helper/namespace_helper.cpp b/src/mds/nameserver2/helper/namespace_helper.cpp index 5eb2bfd4f8..4a27952864 100644 --- a/src/mds/nameserver2/helper/namespace_helper.cpp +++ b/src/mds/nameserver2/helper/namespace_helper.cpp @@ -23,6 +23,7 @@ #include #include "src/mds/nameserver2/helper/namespace_helper.h" #include "src/common/string_util.h" +#include "src/common/timeutility.h" #include "src/common/namespace_define.h" using ::curve::common::COMMON_PREFIX_LENGTH; @@ -31,6 +32,9 @@ using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX; using ::curve::common::SEGMENTKEYLEN; using ::curve::common::SEGMENTINFOKEYPREFIX; using ::curve::common::SEGMENTALLOCSIZEKEY; +using ::curve::common::DISCARDSEGMENTKEYLEN; +using ::curve::common::DISCARDSEGMENTKEYPREFIX; +using ::curve::common::DISCARDSEGMENTKEYEND; namespace curve { namespace mds { @@ -68,6 +72,18 @@ std::string NameSpaceStorageCodec::EncodeSegmentStoreKey(uint64_t inodeID, return storeKey; } +std::string NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey( + const InodeID inodeId, const uint64_t offset) { + std::string storeKey; + storeKey.resize(DISCARDSEGMENTKEYLEN); + ::memcpy(&(storeKey[0]), DISCARDSEGMENTKEYPREFIX, COMMON_PREFIX_LENGTH); + ::curve::common::EncodeBigEndian(&(storeKey[2]), inodeId); + ::curve::common::EncodeBigEndian(&(storeKey[10]), offset); + ::curve::common::EncodeBigEndian( + &(storeKey[18]), curve::common::TimeUtility::GetTimeofDayUs()); + return storeKey; +} + bool NameSpaceStorageCodec::EncodeFileInfo(const FileInfo &fileInfo, std::string *out) { return fileInfo.SerializeToString(out); @@ -132,5 +148,16 @@ bool NameSpaceStorageCodec::DecodeSegmentAllocValue( return true; } + +bool NameSpaceStorageCodec::EncodeDiscardSegment(const DiscardSegmentInfo& info, + std::string* out) { + return info.SerializeToString(out); +} + +bool NameSpaceStorageCodec::DecodeDiscardSegment( + const std::string& info, DiscardSegmentInfo* discardSegmentInfo) { + return discardSegmentInfo->ParseFromString(info); +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/helper/namespace_helper.h b/src/mds/nameserver2/helper/namespace_helper.h index b3e8a74f71..01eba19972 100644 --- a/src/mds/nameserver2/helper/namespace_helper.h +++ b/src/mds/nameserver2/helper/namespace_helper.h @@ -38,6 +38,8 @@ class NameSpaceStorageCodec { static std::string EncodeSnapShotFileStoreKey(uint64_t parentID, const std::string &fileName); static std::string EncodeSegmentStoreKey(uint64_t inodeID, offset_t offset); + static std::string EncodeDiscardSegmentStoreKey(const InodeID inodeId, + const uint64_t offset); static bool EncodeFileInfo(const FileInfo &finlInfo, std::string *out); static bool DecodeFileInfo(const std::string info, FileInfo *fileInfo); @@ -46,6 +48,11 @@ class NameSpaceStorageCodec { static std::string EncodeID(uint64_t value); static bool DecodeID(const std::string &value, uint64_t *out); + static bool EncodeDiscardSegment(const DiscardSegmentInfo& info, + std::string* out); + static bool DecodeDiscardSegment(const std::string& info, + DiscardSegmentInfo* discardSegmentInfo); + static std::string EncodeSegmentAllocKey(uint16_t lid); static std::string EncodeSegmentAllocValue(uint16_t lid, uint64_t alloc); static bool DecodeSegmentAllocValue( diff --git a/src/mds/nameserver2/namespace_service.cpp b/src/mds/nameserver2/namespace_service.cpp index 534a7f14dc..bf3280712f 100644 --- a/src/mds/nameserver2/namespace_service.cpp +++ b/src/mds/nameserver2/namespace_service.cpp @@ -355,6 +355,85 @@ void NameSpaceService::GetOrAllocateSegment( return; } +void NameSpaceService::DeAllocateSegment( + ::google::protobuf::RpcController* controller, + const ::curve::mds::DeAllocateSegmentRequest* request, + ::curve::mds::DeAllocateSegmentResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + butil::Timer timer; + timer.start(); + + if (!isPathValid(request->filename())) { + response->set_statuscode(StatusCode::kParaError); + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment request path is invalid, " + << request->ShortDebugString(); + return; + } + + LOG(INFO) << "logid = " << cntl->log_id() << ", DeAllocateSegment request, " + << request->ShortDebugString(); + + FileWriteLockGuard guard(fileLockManager_, request->filename()); + + std::string signature; + if (request->has_signature()) { + signature = request->signature(); + } + + StatusCode retCode; + retCode = kCurveFS.CheckFileOwner(request->filename(), request->owner(), + signature, request->date()); + if (retCode != StatusCode::kOK) { + response->set_statuscode(retCode); + if (google::ERROR != GetMdsLogLevel(retCode)) { + LOG(WARNING) << "logid = " << cntl->log_id() + << ", DeAllocateSegment CheckFileOwner fail, " + << request->ShortDebugString() + << ", retCode = " << retCode; + } else { + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment CheckFileOwner fail, " + << request->ShortDebugString() + << ", retCode = " << retCode; + } + + return; + } + + retCode = + kCurveFS.DeAllocateSegment(request->filename(), request->offset()); + + timer.stop(); + if (retCode != StatusCode::kOK) { + response->set_statuscode(retCode); + if (google::ERROR != GetMdsLogLevel(retCode)) { + LOG(WARNING) << "logid = " << cntl->log_id() + << ", DeAllocateSegment fail, " + << request->ShortDebugString() + << ", statusCode = " << retCode + << ", StatusCode_Name = " << StatusCode_Name(retCode) + << ", cost " << timer.m_elapsed(0.0) << " ms"; + } else { + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment fail, " + << request->ShortDebugString() + << ", statusCode = " << retCode + << ", StatusCode_Name = " << StatusCode_Name(retCode) + << ", cost " << timer.m_elapsed(0.0) << " ms"; + } + } else { + response->set_statuscode(StatusCode::kOK); + LOG(INFO) << "logid = " << cntl->log_id() << ", DeAllocateSegment ok, " + << request->ShortDebugString() << ", cost " + << timer.m_elapsed(0.0) << " ms"; + } + + return; +} + void NameSpaceService::RenameFile(::google::protobuf::RpcController* controller, const ::curve::mds::RenameFileRequest* request, ::curve::mds::RenameFileResponse* response, diff --git a/src/mds/nameserver2/namespace_service.h b/src/mds/nameserver2/namespace_service.h index 2f4c667bde..07dbba420d 100644 --- a/src/mds/nameserver2/namespace_service.h +++ b/src/mds/nameserver2/namespace_service.h @@ -91,6 +91,12 @@ class NameSpaceService: public CurveFSService { ::curve::mds::GetOrAllocateSegmentResponse* response, ::google::protobuf::Closure* done) override; + void DeAllocateSegment( + ::google::protobuf::RpcController* controller, + const ::curve::mds::DeAllocateSegmentRequest* request, + ::curve::mds::DeAllocateSegmentResponse* response, + ::google::protobuf::Closure* done) override; + void RenameFile(::google::protobuf::RpcController* controller, const ::curve::mds::RenameFileRequest* request, ::curve::mds::RenameFileResponse* response, diff --git a/src/mds/nameserver2/namespace_storage.cpp b/src/mds/nameserver2/namespace_storage.cpp index c65dfb46a2..92de6af4a6 100644 --- a/src/mds/nameserver2/namespace_storage.cpp +++ b/src/mds/nameserver2/namespace_storage.cpp @@ -21,12 +21,15 @@ */ #include +#include #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/nameserver2/helper/namespace_helper.h" #include "src/common/namespace_define.h" using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX; using ::curve::common::SNAPSHOTFILEINFOKEYEND; +using ::curve::common::DISCARDSEGMENTKEYPREFIX; +using ::curve::common::DISCARDSEGMENTKEYEND; namespace curve { namespace mds { @@ -525,6 +528,89 @@ StoreStatus NameServerStorageImp::DeleteSegment( return getErrorCode(errCode); } +StoreStatus NameServerStorageImp::ListDiscardSegment( + std::map* discardSegments) { + assert(discardSegments != nullptr); + + std::vector> out; + int err = + client_->List(DISCARDSEGMENTKEYPREFIX, DISCARDSEGMENTKEYEND, &out); + if (err != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "ListDiscardSegment return error, err = " << err; + return StoreStatus::InternalError; + } + + for (const auto& kv : out) { + DiscardSegmentInfo info; + if (!NameSpaceStorageCodec::DecodeDiscardSegment(kv.second, &info)) { + LOG(ERROR) << "Decode DiscardSegment failed"; + return StoreStatus::InternalError; + } + + discardSegments->emplace(kv.first, std::move(info)); + } + + return StoreStatus::OK; +} + +StoreStatus NameServerStorageImp::DiscardSegment( + const FileInfo& fileInfo, const PageFileSegment& segment) { + const uint64_t inodeId = fileInfo.id(); + const uint64_t offset = segment.startoffset(); + const std::string segmentKey = + NameSpaceStorageCodec::EncodeSegmentStoreKey(inodeId, offset); + const std::string cleanSegmentKey = + NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey(inodeId, offset); + + std::string encodeSegment; + if (!NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)) { + return StoreStatus::InternalError; + } + + std::string encodeDiscardSegment; + DiscardSegmentInfo discardInfo; + discardInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardInfo.set_allocated_pagefilesegment(new PageFileSegment(segment)); + if (!NameSpaceStorageCodec::EncodeDiscardSegment(discardInfo, + &encodeDiscardSegment)) { + return StoreStatus::InternalError; + } + + Operation op1{ + OpType::OpDelete, + const_cast(segmentKey.c_str()), + const_cast(encodeSegment.c_str()), + segmentKey.size(), encodeSegment.size()}; + Operation op2{ + OpType::OpPut, + const_cast(cleanSegmentKey.c_str()), + const_cast(encodeDiscardSegment.c_str()), + cleanSegmentKey.size(), encodeDiscardSegment.size()}; + + std::vector ops{op1, op2}; + auto errCode = client_->TxnN(ops); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "Discard segment failed, filename: " + << fileInfo.filename() << ", inodeid = " << inodeId + << ", offset: " << offset << ", errCode: " << errCode; + } else { + cache_->Remove(segmentKey); + } + + return getErrorCode(errCode); +} + +StoreStatus NameServerStorageImp::CleanDiscardSegment(const std::string& key, + int64_t* revision) { + int errCode = client_->DeleteRewithRevision(key, revision); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "CleanDiscardSegment failed, key = " << key + << ", err = " << errCode; + } + + return getErrorCode(errCode); +} + StoreStatus NameServerStorageImp::SnapShotFile(const FileInfo *originFInfo, const FileInfo *snapshotFInfo) { std::string originFileKey; diff --git a/src/mds/nameserver2/namespace_storage.h b/src/mds/nameserver2/namespace_storage.h index f30497843f..7229ab1726 100644 --- a/src/mds/nameserver2/namespace_storage.h +++ b/src/mds/nameserver2/namespace_storage.h @@ -220,6 +220,35 @@ class NameServerStorage { virtual StoreStatus DeleteSegment( InodeID id, uint64_t off, int64_t *revision) = 0; + /** + * @brief Move segment metadata from SegmentTable to DiscardSegmentTable, + * another background task will delete all chunks and delete segment + * in DiscardSegmentTable + * @param[in] id: Inode ID of the target file + * @param[in] off: Offset of the target segment + * + * @return StoreStatus: error code + */ + virtual StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) = 0; + + /** + * @brief Remove discard segment from DiscardSegmentTable + * @param[in] key discard segment's key + * @param[out] revision: the version number of this operation + * @return On success, return StoreStatus::OK + */ + virtual StoreStatus CleanDiscardSegment(const std::string& key, + int64_t* revision) = 0; + + /** + * @brief list all segment from DiscardSegmentTable + * @param[out] store key and values + * @return On success, return StoreStatus::OK + */ + virtual StoreStatus ListDiscardSegment( + std::map* out) = 0; + /** * @brief SnapShotFile: Transaction for storing metadata of snapshotFile, * and update source file metadata @@ -295,6 +324,15 @@ class NameServerStorageImp : public NameServerStorage { StoreStatus DeleteSegment( InodeID id, uint64_t off, int64_t *revision) override; + StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) override; + + StoreStatus CleanDiscardSegment(const std::string& key, + int64_t* revision) override; + + StoreStatus ListDiscardSegment( + std::map* out) override; + StoreStatus SnapShotFile(const FileInfo *originalFileInfo, const FileInfo * snapshotFileInfo) override; diff --git a/src/mds/server/mds.cpp b/src/mds/server/mds.cpp index 6a3bc05e96..6a99c73cd8 100644 --- a/src/mds/server/mds.cpp +++ b/src/mds/server/mds.cpp @@ -144,6 +144,9 @@ void MDS::Run() { LOG_IF(FATAL, !cleanManager_->Start()) << "start cleanManager fail."; // recover unfinished tasks cleanManager_->RecoverCleanTasks(); + + cleanDiscardSegmentTask_->Start(); + // start scheduler module coordinator_->Run(); // start brpc server @@ -166,6 +169,8 @@ void MDS::Stop() { kCurveFS.Uninit(); + cleanDiscardSegmentTask_->Stop(); + cleanManager_->Stop(); topologyMetricService_->Stop(); @@ -419,6 +424,17 @@ void MDS::InitCurveFS(const CurveFSOption& curveFSOptions) { // init clean manager InitCleanManager(); + // init clean discard segment task + uint32_t scanDiscardTaskIntervalMs = 0; + if (!conf_->GetUInt32Value("mds.segment.discard.scanIntevalMs", + &scanDiscardTaskIntervalMs)) { + LOG(FATAL) << "Load mds.segment.discard.scanIntevalMs from config " + "file failed"; + } + + cleanDiscardSegmentTask_ = std::make_shared( + cleanManager_, nameServerStorage_, scanDiscardTaskIntervalMs); + // init FileRecordManager auto fileRecordManager = std::make_shared(); diff --git a/src/mds/server/mds.h b/src/mds/server/mds.h index c4b1ed9289..815272bfaa 100644 --- a/src/mds/server/mds.h +++ b/src/mds/server/mds.h @@ -223,6 +223,7 @@ class MDS { std::shared_ptr topologyMetricService_; std::shared_ptr topologyServiceManager_; std::shared_ptr cleanManager_; + std::shared_ptr cleanDiscardSegmentTask_; std::shared_ptr coordinator_; std::shared_ptr heartbeatManager_; char* etcdEndpoints_; diff --git a/test/client/BUILD b/test/client/BUILD index 826ec8f67f..ad29983581 100644 --- a/test/client/BUILD +++ b/test/client/BUILD @@ -73,6 +73,7 @@ cc_test( "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", "//test/integration/cluster_common:integration_cluster_common", + "//test/client/mock:client_mock_lib", ], ) @@ -187,6 +188,7 @@ cc_test( "//proto:topology_cc_proto", "//src/client:curve_client", "//src/common:curve_common", + "//src/client:cbd", "//test/client/fake:fake_lib", "//src/common/concurrent:curve_concurrent", "@com_google_googletest//:gtest", diff --git a/test/client/client_common_unittest.cpp b/test/client/client_common_unittest.cpp index e09aa36a84..d0faf57d82 100644 --- a/test/client/client_common_unittest.cpp +++ b/test/client/client_common_unittest.cpp @@ -24,8 +24,8 @@ #include "src/client/client_common.h" -using curve::client::EndPoint; -using curve::client::ChunkServerAddr; +namespace curve { +namespace client { TEST(ClientCommon, ChunkServerAddrTest) { // 默认构造函数创建的成员变量内容为空 @@ -70,3 +70,6 @@ TEST(ClientCommon, ChunkServerAddrTest) { str2endpoint("127.0.0.1:9000", &ep1); ASSERT_EQ(caddr2.addr_, ep1); } + +} // namespace client +} // namespace curve diff --git a/test/client/client_mdsclient_metacache_unittest.cpp b/test/client/client_mdsclient_metacache_unittest.cpp index a8a82387fd..a75cc11b50 100644 --- a/test/client/client_mdsclient_metacache_unittest.cpp +++ b/test/client/client_mdsclient_metacache_unittest.cpp @@ -82,20 +82,20 @@ class MDSClientTest : public ::testing::Test { userinfo.owner = "test"; if (server.AddService(&topologyservice, - brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { - LOG(FATAL) << "Fail to add service"; + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + ASSERT_TRUE(false) << "Fail to add service"; } if (server.AddService(&curvefsservice, - brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { - LOG(FATAL) << "Fail to add service"; + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + ASSERT_TRUE(false) << "Fail to add service"; } - curve::mds::topology::GetChunkServerInfoResponse* response - = new curve::mds::topology::GetChunkServerInfoResponse(); + curve::mds::topology::GetChunkServerInfoResponse* response = + new curve::mds::topology::GetChunkServerInfoResponse(); response->set_statuscode(0); - curve::mds::topology::ChunkServerInfo* serverinfo - = new curve::mds::topology::ChunkServerInfo(); + curve::mds::topology::ChunkServerInfo* serverinfo = + new curve::mds::topology::ChunkServerInfo(); serverinfo->set_chunkserverid(888); serverinfo->set_disktype("nvme"); serverinfo->set_hostip("10.182.26.2"); @@ -116,7 +116,6 @@ class MDSClientTest : public ::testing::Test { LOG(INFO) << "meta server addr = " << mdsMetaServerAddr.c_str(); ASSERT_EQ(server.Start(mdsMetaServerAddr.c_str(), &options), 0); - LOG(INFO) << configpath.c_str(); ASSERT_EQ(0, Init(configpath.c_str())) << "Fail to init config, path = " << configpath; } @@ -2205,6 +2204,67 @@ TEST_F(MDSClientTest, StatFileStatusTest) { } } +TEST_F(MDSClientTest, DeAllocateSegmentTest) { + FInfo fileInfo; + fileInfo.fullPathName = "/DeAllocateSegmentTest"; + + // rpc failed + { + brpc::Controller cntl; + cntl.SetFailed(-1, "rpc failed"); + + FakeReturn* fakeRet = new FakeReturn(&cntl, nullptr); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + uint64_t startMs = curve::common::TimeUtility::GetTimeofDayMs(); + ASSERT_EQ(LIBCURVE_ERROR::FAILED, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + uint64_t endMs = curve::common::TimeUtility::GetTimeofDayMs(); + ASSERT_GE(endMs - startMs, metaopt.mdsMaxRetryMS); + } + + // rpc return ok + { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(curve::mds::StatusCode::kOK); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + + // rpc return segment not allocated + { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(curve::mds::StatusCode::kSegmentNotAllocated); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + + // other error code + { + std::vector errorCodes{ + curve::mds::StatusCode::kOwnerAuthFail, + curve::mds::StatusCode::kParaError, + curve::mds::StatusCode::kNotSupported, + curve::mds::StatusCode::kFileUnderSnapShot}; + + for (auto err : errorCodes) { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(err); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_NE(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + } +} + using ::testing::_; using ::testing::DoAll; using ::testing::ElementsAre; diff --git a/test/client/client_unittest_main.cpp b/test/client/client_unittest_main.cpp index 47c3663bc9..6d03a8a4bf 100644 --- a/test/client/client_unittest_main.cpp +++ b/test/client/client_unittest_main.cpp @@ -20,32 +20,13 @@ * Author: tongguangxun */ - -#include -#include #include -#include -#include #include -#include -#include #include -#include #include +#include -#include "include/curve_compiler_specific.h" -#include "proto/nameserver2.pb.h" -#include "proto/topology.pb.h" -#include "src/client/io_condition_varaiable.h" -#include "src/client/iomanager4file.h" -#include "src/client/io_tracker.h" -#include "src/client/client_common.h" -#include "src/client/metacache.h" -#include "src/client/request_context.h" #include "src/client/file_instance.h" -#include "src/client/splitor.h" -#include "src/client/libcurve_file.h" -#include "include/client/libcurve.h" #include "test/integration/cluster_common/cluster.h" #include "test/util/config_generator.h" diff --git a/test/client/client_userifo_unittest.cpp b/test/client/client_userinfo_unittest.cpp similarity index 100% rename from test/client/client_userifo_unittest.cpp rename to test/client/client_userinfo_unittest.cpp diff --git a/test/client/copyset_client_test.cpp b/test/client/copyset_client_test.cpp index 9b64cfa1ab..c7c585abe5 100644 --- a/test/client/copyset_client_test.cpp +++ b/test/client/copyset_client_test.cpp @@ -3549,11 +3549,12 @@ TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { const std::string& configPath = "./conf/client.conf"; cc.Init(configPath.c_str()); FileInstance fileinstance; - UserInfo_t userinfo; + UserInfo userinfo; userinfo.owner = "userinfo"; MDSClient mdsclient; - mdsclient.Initialize(cc.GetFileServiceOption().metaServerOpt); + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient.Initialize(cc.GetFileServiceOption().metaServerOpt)); ASSERT_TRUE(fileinstance.Initialize("/test", &mdsclient, userinfo, cc.GetFileServiceOption())); @@ -3573,8 +3574,8 @@ TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { << "Fail to start server add 127.0.0.1:9102"; // fill metacache - curve::client::MetaCache* mc - = fileinstance.GetIOManager4File()->GetMetaCache(); + curve::client::MetaCache* mc = + fileinstance.GetIOManager4File()->GetMetaCache(); curve::client::ChunkIDInfo_t chunkinfo(1, 2, 3); mc->UpdateChunkInfoByIndex(0, chunkinfo); curve::client::CopysetInfo cpinfo; diff --git a/test/client/discard_task_test.cpp b/test/client/discard_task_test.cpp new file mode 100644 index 0000000000..48817f1279 --- /dev/null +++ b/test/client/discard_task_test.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Sat Dec 19 22:49:56 CST 2020 + */ + +#include "src/client/discard_task.h" + +#include +#include + +#include + +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock_meta_cache.h" + +namespace curve { +namespace client { + +using ::testing::Return; + +class DiscardTaskTest : public ::testing::Test { + public: + void SetUp() override { + mockMetaCache_.reset(new MockMetaCache()); + mockMDSClient_.reset(new MockMDSClient()); + + fileInfo_.fullPathName = "/TestDiscardTask"; + fileInfo_.chunksize = 16ull * 1024 * 1024; + fileInfo_.segmentsize = 1ull * 1024 * 1024 * 1024; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + } + + protected: + FInfo fileInfo_; + DiscardTaskManager discarTaskManager_; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; +}; + +TEST_F(DiscardTaskTest, TestDiscardBitmapCleared) { + SegmentIndex segmentIndex = 100; + uint64_t offset = segmentIndex * 1024ull * 1024 * 1024; + DiscardTask task(&discarTaskManager_, segmentIndex, mockMetaCache_.get(), + mockMDSClient_.get()); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .Times(0); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(0); + + ASSERT_NO_FATAL_FAILURE(task.Run()); +} + +TEST_F(DiscardTaskTest, TestAllDiscard) { + SegmentIndex segmentIndex = 100; + uint64_t offset = segmentIndex * 1024ull * 1024 * 1024; + DiscardTask task(&discarTaskManager_, segmentIndex, mockMetaCache_.get(), + mockMDSClient_.get()); + + // mdsclient return OK + { + // set all bit + FileSegment* segment = + mockMetaCache_->GetFileSegment(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(1); + + ASSERT_NO_FATAL_FAILURE(task.Run()); + ASSERT_FALSE(segment->IsAllDiscard()); + } + + // mdsclient return failed + { + // set all bit + FileSegment* segment = + mockMetaCache_->GetFileSegment(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(0); + + ASSERT_NO_FATAL_FAILURE(task.Run()); + ASSERT_TRUE(segment->IsAllDiscard()); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/fake/fakeMDS.cpp b/test/client/fake/fakeMDS.cpp index 53ab4c0be5..eddbf621e4 100644 --- a/test/client/fake/fakeMDS.cpp +++ b/test/client/fake/fakeMDS.cpp @@ -150,6 +150,15 @@ bool FakeMDS::StartService() { FakeReturn* fakeGetFileInforet = new FakeReturn(nullptr, static_cast(getfileinforesponse)); // NOLINT fakecurvefsservice_.SetGetFileInfoFakeReturn(fakeGetFileInforet); + // set DeAllocateSegment fake return + auto* deAllocateSegmentResponse = + new curve::mds::DeAllocateSegmentResponse(); + deAllocateSegmentResponse->set_statuscode(curve::mds::StatusCode::kOK); + auto* deAllocateSegmentFakeRet = + new FakeReturn(nullptr, deAllocateSegmentResponse); + fakecurvefsservice_.SetDeAllocateSegmentFakeReturn( + deAllocateSegmentFakeRet); + /** * set GetOrAllocateSegment fake return */ diff --git a/test/client/fake/fakeMDS.h b/test/client/fake/fakeMDS.h index f957138e1f..e0206e7c39 100644 --- a/test/client/fake/fakeMDS.h +++ b/test/client/fake/fakeMDS.h @@ -196,6 +196,22 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { response->CopyFrom(*resp); } + void DeAllocateSegment(google::protobuf::RpcController* cntl_base, + const curve::mds::DeAllocateSegmentRequest* request, + curve::mds::DeAllocateSegmentResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + if (fakeDeAllocateSegment_->controller_ != nullptr && + fakeDeAllocateSegment_->controller_->Failed()) { + cntl_base->SetFailed("failed"); + return; + } + + auto fakeResponse = + static_cast(fakeDeAllocateSegment_->response_); + response->CopyFrom(*fakeResponse); + } + void OpenFile(::google::protobuf::RpcController* controller, const ::curve::mds::OpenFileRequest* request, ::curve::mds::OpenFileResponse* response, @@ -601,6 +617,10 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { fakeGetOrAllocateSegmentret_ = fakeret; } + void SetDeAllocateSegmentFakeReturn(FakeReturn* fakeret) { + fakeDeAllocateSegment_ = fakeret; + } + void SetOpenFile(FakeReturn* fakeret) { fakeopenfile_ = fakeret; } @@ -707,6 +727,7 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { FakeReturn* fakeGetFileInforet_; FakeReturn* fakeGetAllocatedSizeRet_; FakeReturn* fakeGetOrAllocateSegmentret_; + FakeReturn* fakeDeAllocateSegment_; FakeReturn* fakeopenfile_; FakeReturn* fakeclosefile_; FakeReturn* fakerenamefile_; diff --git a/test/client/file_instance_test.cpp b/test/client/file_instance_test.cpp index 5fa91c0686..eeaf2948e2 100644 --- a/test/client/file_instance_test.cpp +++ b/test/client/file_instance_test.cpp @@ -59,5 +59,24 @@ TEST(FileInstanceTest, CommonTest) { fi4.UnInitialize(); } +TEST(FileInstanceTest, OpenReadonlyAndDiscardTest) { + FileInstance instance; + FileServiceOption opt; + MDSClient mdsClient; + UserInfo userInfo{"hello", "world"}; + + ASSERT_TRUE( + instance.Initialize("/FileInstanceTest-OpenReadonlyAndDiscardTest", + &mdsClient, userInfo, opt, true)); + + ASSERT_EQ(-1, instance.Discard(0, 0)); + + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP::LIBCURVE_OP_DISCARD; + aioctx.offset = 0; + aioctx.length = 0; + ASSERT_EQ(-1, instance.AioDiscard(&aioctx)); +} + } // namespace client } // namespace curve diff --git a/test/client/file_segment_test.cpp b/test/client/file_segment_test.cpp new file mode 100644 index 0000000000..1622f60126 --- /dev/null +++ b/test/client/file_segment_test.cpp @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Tue Dec 15 20:38:06 CST 2020 + * Author: wuhanqing + */ + +#include +#include + +#include "src/client/metacache_struct.h" + +namespace curve { +namespace client { + +TEST(FileSegmentTest, TestDiscard) { + const SegmentIndex segmentIndex = 0; + const uint32_t segmentSize = 1 * GiB; + + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (const auto& discardGranularity : discardGranularities) { + LOG(INFO) << "discardGranularity: " << discardGranularity; + + // discard entire segment + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetDiscard(0, segmentSize); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextClearBit(0)); + } + + // discard length smaller than discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetDiscard(0, discardGranularity - 1); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + + segment.SetDiscard(segmentSize - discardGranularity, 1); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } + + // discard offset and length are both align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetDiscard(0, discardGranularity); + segment.SetDiscard(25 * discardGranularity, 4 * discardGranularity); + segment.SetDiscard(segmentSize - 2 * discardGranularity, + 2 * discardGranularity); + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : setRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {0, 0}, {25, 28}, {endIndex - 1, endIndex}}; + + ASSERT_EQ(setRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < setRanges.size(); ++i) { + ASSERT_EQ(setRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(setRanges[i].endIndex, expectedRanges[i].endIndex); + } + } + + // discard offset or length aren't align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetDiscard(discardGranularity, + discardGranularity + 1); // {1, 1} + segment.SetDiscard(5 * discardGranularity - 1, + discardGranularity + 1); // {5, 5} + segment.SetDiscard(10 * discardGranularity + 1, + 2 * discardGranularity - 2); // not valid + segment.SetDiscard(20 * discardGranularity + 1, + 3 * discardGranularity - 2); // {21, 21} + segment.SetDiscard(30 * discardGranularity - 1, + discardGranularity + 2); // {30, 30} + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : setRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {1, 1}, {5, 5}, {21, 21}, {30, 30}}; + + ASSERT_EQ(setRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < setRanges.size(); ++i) { + ASSERT_EQ(setRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(setRanges[i].endIndex, expectedRanges[i].endIndex); + } + + segment.SetDiscard(1, segmentSize - 2); + clearRanges.clear(); + setRanges.clear(); + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + ASSERT_EQ(1, setRanges.size()); + ASSERT_EQ(1, setRanges.front().beginIndex); + ASSERT_EQ(endIndex - 1, setRanges.front().endIndex); + } + } +} + +TEST(FileSegmentTest, TestClearDiscard) { + const SegmentIndex segmentIndex = 0; + const uint32_t segmentSize = 1 * GiB; + + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (const auto& discardGranularity : discardGranularities) { + LOG(INFO) << "discardGranularity: " << discardGranularity; + + // clear entire segment + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(0, segmentSize); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } + + // clear length smaller than discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(0, discardGranularity - 1); + ASSERT_FALSE(segment.GetBitmap().Test(0)); + + segment.ClearDiscard(segmentSize - discardGranularity, 1); + ASSERT_FALSE( + segment.GetBitmap().Test(segmentSize / discardGranularity - 1)); + } + + // clear offset and length are both align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(0, discardGranularity); + segment.ClearDiscard(25 * discardGranularity, + 4 * discardGranularity); + segment.ClearDiscard(segmentSize - 2 * discardGranularity, + 2 * discardGranularity); + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : clearRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {0, 0}, {25, 28}, {endIndex - 1, endIndex}}; + + ASSERT_EQ(clearRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < clearRanges.size(); ++i) { + ASSERT_EQ(clearRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(clearRanges[i].endIndex, expectedRanges[i].endIndex); + } + } + + // clear offset or length aren't align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearDiscard(discardGranularity, + discardGranularity + 1); // {1, 2} + segment.ClearDiscard(5 * discardGranularity - 1, + discardGranularity + 1); // {4, 5} + segment.ClearDiscard(10 * discardGranularity + 1, + 2 * discardGranularity - 2); // {10, 11} + segment.ClearDiscard(20 * discardGranularity + 1, + 3 * discardGranularity - 2); // {20, 22} + segment.ClearDiscard(30 * discardGranularity - 1, + discardGranularity + 2); // {29, 31} + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : clearRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {1, 2}, {4, 5}, {10, 11}, {20, 22}, {29, 31}}; + + ASSERT_EQ(clearRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < clearRanges.size(); ++i) { + ASSERT_EQ(clearRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(clearRanges[i].endIndex, expectedRanges[i].endIndex); + } + + segment.GetBitmap().Set(); + clearRanges.clear(); + setRanges.clear(); + segment.ClearDiscard(1, segmentSize - 2); + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + ASSERT_EQ(1, clearRanges.size()); + ASSERT_EQ(0, clearRanges.front().beginIndex); + ASSERT_EQ(endIndex, clearRanges.front().endIndex); + } + } +} + +TEST(FileSegmentTest, TestMultiSetDiscard) { + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (auto discardGranularity : discardGranularities) { + FileSegment segment(50, 1 * GiB, discardGranularity); + + uint64_t offset = 0ull; + uint64_t length = 32 * MiB; + while (offset < 1 * GiB) { + segment.SetDiscard(offset, length); + offset += length; + } + + ASSERT_TRUE(segment.IsAllDiscard()); + } +} + +TEST(FileSegmentTest, TestMultiClearDiscard) { + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (auto discardGranularity : discardGranularities) { + FileSegment segment(50, 1 * GiB, discardGranularity); + segment.GetBitmap().Set(); + + uint64_t offset = 0ull; + uint64_t length = 32 * MiB; + while (offset < 1 * GiB) { + segment.ClearDiscard(offset, length); + offset += length; + } + + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/iotracker_splitor_unittest.cpp b/test/client/iotracker_splitor_unittest.cpp index a976396020..2d3f654298 100644 --- a/test/client/iotracker_splitor_unittest.cpp +++ b/test/client/iotracker_splitor_unittest.cpp @@ -879,14 +879,13 @@ TEST_F(IOTrackerSplitorTest, InvalidParam) { iotracker, mc, &reqlist, nullptr, offset, length, &mdsclient_, &fi)); + ASSERT_EQ(-1, Splitor::CalcDiscardSegments(nullptr)); + delete iotracker; delete[] buf; } -TEST(SplitorTest, RequestSourceInfoTest) { - using curve::client::ChunkIndex; - using curve::client::RequestSourceInfo; - +TEST_F(IOTrackerSplitorTest, RequestSourceInfoTest) { IOTracker ioTracker(nullptr, nullptr, nullptr); ioTracker.SetOpType(OpType::READ); diff --git a/test/client/iotracker_test.cpp b/test/client/iotracker_test.cpp new file mode 100644 index 0000000000..af296759e8 --- /dev/null +++ b/test/client/iotracker_test.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Sun Dec 20 10:25:08 CST 2020 + */ + +#include +#include +#include + +#include // NOLINT +#include +#include // NOLINT + +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock_meta_cache.h" + +namespace curve { +namespace client { + +using ::testing::Return; +using ::testing::Matcher; +using ::testing::AllOf; +using ::testing::Ge; +using ::testing::Le; + +class IOTrackerTest : public ::testing::Test { + public: + void SetUp() override { + opt_.discardTaskDelayMs = 10; + IOTracker::InitDiscardOption(opt_); + + mockMetaCache_.reset(new MockMetaCache()); + mockMDSClient_.reset(new MockMDSClient()); + + fileInfo_.fullPathName = "/IOTrackerTest"; + fileInfo_.length = 100 * GiB; + fileInfo_.segmentsize = 1 * GiB; + fileInfo_.chunksize = 16 * MiB; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + } + + void TearDown() override { + discardTaskManager_.Stop(); + } + + protected: + DiscardOption opt_; + FInfo fileInfo_; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; + DiscardTaskManager discardTaskManager_; +}; + +TEST_F(IOTrackerTest, TestDiscardNotSatisfyOneSegment) { + IOTracker iotracker(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset = 50 * GiB; + uint32_t length = 512 * MiB; + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, _)) + .Times(0); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(_)) + .Times(0); + + iotracker.StartDiscard(offset, length, mockMDSClient_.get(), &fileInfo_, + &discardTaskManager_); + ASSERT_EQ(length, iotracker.Wait()); +} + +TEST_F(IOTrackerTest, TestDiscardOneSegment) { + IOTracker iotracker(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset = 50 * GiB; + uint32_t length = 1 * GiB; + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, 50 * GiB)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(50)) + .Times(1); + + iotracker.StartDiscard(offset, length, mockMDSClient_.get(), &fileInfo_, + &discardTaskManager_); + ASSERT_EQ(length, iotracker.Wait()); + + std::this_thread::sleep_for( + std::chrono::milliseconds(5 * opt_.discardTaskDelayMs)); +} + +TEST_F(IOTrackerTest, TestDiscardMultiSegment) { + // discard three times + IOTracker iotracker1(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset1 = 50 * GiB; + uint32_t length1 = 512 * MiB; + + IOTracker iotracker2(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset2 = 52 * GiB + 512 * MiB; + uint32_t length2 = 512 * MiB; + + IOTracker iotracker3(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset3 = 50 * GiB + 512 * MiB; + uint32_t length3 = 2 * GiB; + + Matcher offsetRange = AllOf(Ge(50 * GiB), Le(52 * GiB)); + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offsetRange)) + .Times(3) + .WillRepeatedly(Return(LIBCURVE_ERROR::OK)); + + Matcher segmentIndexRange = AllOf(Ge(50), Le(52)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndexRange)) + .Times(3); + + iotracker1.StartDiscard(offset1, length1, mockMDSClient_.get(), &fileInfo_, + &discardTaskManager_); + ASSERT_EQ(length1, iotracker1.Wait()); + + iotracker2.StartDiscard(offset2, length2, mockMDSClient_.get(), &fileInfo_, + &discardTaskManager_); + ASSERT_EQ(length2, iotracker2.Wait()); + + iotracker3.StartDiscard(offset3, length3, mockMDSClient_.get(), &fileInfo_, + &discardTaskManager_); + ASSERT_EQ(length3, iotracker3.Wait()); + + std::this_thread::sleep_for( + std::chrono::milliseconds(5 * opt_.discardTaskDelayMs)); +} + +} // namespace client +} // namespace curve diff --git a/test/client/libcbd_libcurve_test.cpp b/test/client/libcbd_libcurve_test.cpp index 9da87377cf..946a54e6f9 100644 --- a/test/client/libcbd_libcurve_test.cpp +++ b/test/client/libcbd_libcurve_test.cpp @@ -49,6 +49,8 @@ using curve::client::EndPoint; #define filename "1_userinfo_test.img" +const uint64_t GiB = 1024ull * 1024 * 1024; + DECLARE_string(chunkserver_list); extern std::string configpath; @@ -56,6 +58,12 @@ void LibcbdLibcurveTestCallback(CurveAioContext* context) { context->op = LIBCURVE_OP_MAX; } +std::atomic discardComplete(false); +void AioDiscardCallback(CurveAioContext* context) { + ASSERT_EQ(context->ret, context->length); + discardComplete.store(true, std::memory_order_release); +} + class TestLibcbdLibcurve : public ::testing::Test { public: void SetUp() { @@ -200,6 +208,26 @@ TEST_F(TestLibcbdLibcurve, ReadWriteTest) { ASSERT_EQ(ret, LIBCURVE_ERROR::OK); } +TEST_F(TestLibcbdLibcurve, DiscardTest) { + int fd; + CurveOptions opts; + memset(&opts, 0, sizeof(opts)); + + opts.conf = const_cast(configpath.c_str()); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_init(&opts)); + + fd = cbd_lib_open(filename); + ASSERT_GE(fd, 0); + + ASSERT_EQ(4096, cbd_lib_pdiscard(fd, 0, 4096)); + ASSERT_EQ(1 * GiB, cbd_lib_pdiscard(fd, 1 * GiB, 1 * GiB)); + + sleep(1); + + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_close(fd)); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_fini()); +} + TEST_F(TestLibcbdLibcurve, AioReadWriteTest) { int ret; int fd; @@ -259,6 +287,52 @@ TEST_F(TestLibcbdLibcurve, AioReadWriteTest) { ASSERT_EQ(ret, LIBCURVE_ERROR::OK); } +TEST_F(TestLibcbdLibcurve, TestAioDiscard) { + int fd; + CurveOptions opts; + memset(&opts, 0, sizeof(opts)); + opts.conf = const_cast(configpath.c_str()); + + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_init(&opts)); + + fd = cbd_lib_open(filename); + ASSERT_GE(fd, 0); + + { + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP_DISCARD; + aioctx.offset = 0; + aioctx.length = 4096; + aioctx.cb = AioDiscardCallback; + + discardComplete.store(false); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_aio_pdiscard(fd, &aioctx)); + + while (discardComplete.load(std::memory_order_consume) != true) { + usleep(100); + } + } + + { + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP_DISCARD; + aioctx.offset = 1 * GiB; + aioctx.length = 1 * GiB; + aioctx.cb = AioDiscardCallback; + + discardComplete.store(false); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_aio_pdiscard(fd, &aioctx)); + + while (discardComplete.load(std::memory_order_consume) != true) { + usleep(100); + } + } + + sleep(1); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_close(fd)); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_fini()); +} + TEST_F(TestLibcbdLibcurve, StatFileTest) { int64_t ret; CurveOptions opt; @@ -343,6 +417,7 @@ const std::vector clientConf { std::string("metacache.rpcRetryIntervalUS=500"), std::string("mds.rpcRetryIntervalUS=500"), std::string("schedule.threadpoolSize=2"), + std::string("discard.discardTaskDelayMs=10") }; int main(int argc, char ** argv) { diff --git a/test/client/metacache_test.cpp b/test/client/metacache_test.cpp new file mode 100644 index 0000000000..725ac25f51 --- /dev/null +++ b/test/client/metacache_test.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Wed Dec 23 16:21:20 CST 2020 + * Author: wuhanqing + */ + +#include "src/client/metacache.h" + +#include +#include + +#include +#include + +namespace curve { +namespace client { + +class MetaCacheTest : public ::testing::Test { + public: + void SetUp() override {} + + void TearDown() override {} + + protected: + void InsertMetaCache(uint64_t fileLength, uint64_t segmentSize, + uint64_t chunkSize) { + fileInfo_.fullPathName = "/MetaCacheTest"; + fileInfo_.length = fileLength; + fileInfo_.segmentsize = segmentSize; + fileInfo_.chunksize = chunkSize; + + metaCache_.UpdateFileInfo(fileInfo_); + + uint64_t chunks = fileLength / chunkSize; + for (uint64_t i = 0; i < chunks; ++i) { + ChunkIDInfo info(i, i, i); + metaCache_.UpdateChunkInfoByIndex(i, info); + } + } + + uint64_t CountAvaiableChunks(uint64_t fileLength, uint64_t segmentSize, + uint64_t chunksize) { + uint64_t count = 0; + uint64_t chunks = fileLength / chunksize; + ChunkIDInfo info; + + for (uint64_t i = 0; i < chunks; ++i) { + if (MetaCacheErrorType::OK == + metaCache_.GetChunkInfoByIndex(i, &info)) { + ++count; + } + } + + return count; + } + + protected: + FInfo fileInfo_; + MetaCache metaCache_; +}; + +TEST_F(MetaCacheTest, TestCleanChunksInSegment) { + std::vector> testValues{ + {20 * GiB, 1 * GiB, 16 * MiB}, + {20 * GiB, 512 * MiB, 16 * MiB}, + {20 * GiB, 16 * MiB, 16 * MiB}, + {20 * GiB, 16 * MiB, 4 * MiB}, + }; + + for (auto& values : testValues) { + uint64_t fileLength = std::get<0>(values); + uint64_t segmentSize = std::get<1>(values); + uint64_t chunkSize = std::get<2>(values); + + InsertMetaCache(fileLength, segmentSize, chunkSize); + + uint64_t totalChunks = fileLength / chunkSize; + uint64_t totalSegments = fileLength / segmentSize; + uint64_t chunksInSegment = segmentSize / chunkSize; + + ASSERT_EQ(totalChunks, + CountAvaiableChunks(fileLength, segmentSize, chunkSize)); + + SegmentIndex segmentIndex = 0; + uint64_t count = 0; + while (segmentIndex < fileLength / segmentSize) { + metaCache_.CleanChunksInSegment(segmentIndex++); + ++count; + ASSERT_EQ(totalChunks - count * chunksInSegment, + CountAvaiableChunks(fileLength, segmentSize, chunkSize)); + } + + ASSERT_EQ(0, CountAvaiableChunks(fileLength, segmentIndex, chunkSize)); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/mock/BUILD b/test/client/mock/BUILD new file mode 100644 index 0000000000..c9fe6f3b46 --- /dev/null +++ b/test/client/mock/BUILD @@ -0,0 +1,47 @@ +# +# Copyright (c) 2020 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# https://docs.bazel.build/versions/master/be/c-cpp.html#cc_library +COPTS = [ + "-DGFLAGS=gflags", + "-DOS_LINUX", + "-DSNAPPY", + "-DHAVE_SSE42", + "-DNDEBUG", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-msse4.2", + "-pthread", + "-Wsign-compare", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor", + "-Wno-missing-field-initializers", + "-std=c++11", +] + +cc_library( + name="client_mock_lib", + srcs=glob([ + "*.h" + ]), + deps=[ + "//src/client:curve_client", + ], + copts=COPTS, + visibility = ["//visibility:public"], +) diff --git a/test/client/mock/mock_mdsclient.h b/test/client/mock/mock_mdsclient.h new file mode 100644 index 0000000000..3c54cdcc5a --- /dev/null +++ b/test/client/mock/mock_mdsclient.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Sat Dec 19 23:01:09 CST 2020 + */ + +#ifndef TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ +#define TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ + +#include + +#include "src/client/mds_client.h" + +namespace curve { +namespace client { + +class MockMDSClient : public MDSClient { + public: + MOCK_METHOD2(DeAllocateSegment, LIBCURVE_ERROR(const FInfo*, uint64_t)); +}; + +} // namespace client +} // namespace curve + +#endif // TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ diff --git a/test/client/mock_meta_cache.h b/test/client/mock_meta_cache.h index 430bdca640..b002d01c43 100644 --- a/test/client/mock_meta_cache.h +++ b/test/client/mock_meta_cache.h @@ -75,6 +75,8 @@ class MockMetaCache : public MetaCache { &FakeMetaCache::UpdateLeader)); } + MOCK_METHOD1(CleanChunksInSegment, void(SegmentIndex)); + private: FakeMetaCache fakeMetaCache_; }; diff --git a/test/kvstorageclient/etcdclient_test.cpp b/test/kvstorageclient/etcdclient_test.cpp index ac1469a15d..3b25bce259 100644 --- a/test/kvstorageclient/etcdclient_test.cpp +++ b/test/kvstorageclient/etcdclient_test.cpp @@ -182,12 +182,14 @@ TEST_F(TestEtcdClinetImp, test_EtcdClientInterface) { // 3. list file, 可以list到file0~file9 std::vector listRes; - int errCode = client_->List("01", "02", &listRes); + std::vector> listRes2; + int errCode = client_->List("01", "02", &listRes2); ASSERT_EQ(EtcdErrCode::EtcdOK, errCode); - ASSERT_EQ(keyMap.size(), listRes.size()); - for (int i = 0; i < listRes.size(); i++) { + ASSERT_EQ(keyMap.size(), listRes2.size()); + for (int i = 0; i < listRes2.size(); i++) { FileInfo finfo; - ASSERT_TRUE(NameSpaceStorageCodec::DecodeFileInfo(listRes[i], &finfo)); + ASSERT_TRUE( + NameSpaceStorageCodec::DecodeFileInfo(listRes2[i].second, &finfo)); ASSERT_EQ(fileName[i], finfo.filename()); } diff --git a/test/mds/chunkserverclient/test_copyset_client.cpp b/test/mds/chunkserverclient/test_copyset_client.cpp index 44e3a10b57..6cf57a80cc 100644 --- a/test/mds/chunkserverclient/test_copyset_client.cpp +++ b/test/mds/chunkserverclient/test_copyset_client.cpp @@ -34,7 +34,7 @@ #include "src/mds/chunkserverclient/copyset_client.h" #include "test/mds/mock/mock_topology.h" #include "test/mds/mock/mock_chunkserver.h" -#include "test/mds/chunkserverclient/mock_chunkserverclient.h" +#include "test/mds/mock/mock_chunkserverclient.h" #include "src/mds/chunkserverclient/chunkserverclient_config.h" diff --git a/test/mds/chunkserverclient/mock_chunkserverclient.h b/test/mds/mock/mock_chunkserverclient.h similarity index 90% rename from test/mds/chunkserverclient/mock_chunkserverclient.h rename to test/mds/mock/mock_chunkserverclient.h index 85d0da8596..3d4a8582e9 100644 --- a/test/mds/chunkserverclient/mock_chunkserverclient.h +++ b/test/mds/mock/mock_chunkserverclient.h @@ -20,8 +20,8 @@ * Author: xuchaojie */ -#ifndef TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ -#define TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ +#ifndef TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ +#define TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ #include #include "src/mds/chunkserverclient/chunkserver_client.h" @@ -65,4 +65,4 @@ class MockChunkServerClient : public ChunkServerClient { } // namespace mds } // namespace curve -#endif // TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ +#endif // TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ diff --git a/test/mds/mock/mock_etcdclient.h b/test/mds/mock/mock_etcdclient.h index 2d2d1a28ea..a8da477b2c 100644 --- a/test/mds/mock/mock_etcdclient.h +++ b/test/mds/mock/mock_etcdclient.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "src/kvstorageclient/etcd_client.h" #include "src/mds/nameserver2/namespace_storage_cache.h" @@ -42,6 +43,8 @@ class MockEtcdClient : public EtcdClientImp { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp index a2bcc1bbad..11c70f8572 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp @@ -31,6 +31,7 @@ using ::testing::_; using ::testing::Return; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; @@ -44,8 +45,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { { // 1. list失败 - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); std::map out; ASSERT_EQ(-1, AllocStatisticHelper::GetExistSegmentAllocValues( @@ -55,8 +57,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { { // 2. list成功,解析失败 std::vector values{"hello"}; - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); std::map out; @@ -67,8 +70,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { // 3. 获取已有的segment alloc value成功 std::vector values{ NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); std::map out; diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp index 935a79778a..c51e91587c 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp @@ -31,6 +31,7 @@ using ::testing::_; using ::testing::Return; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; @@ -70,8 +71,9 @@ TEST_F(AllocStatisticTest, test_Init) { LOG(INFO) << "test2......"; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(-1, allocStatistic_->Init()); int64_t alloc; @@ -84,8 +86,9 @@ TEST_F(AllocStatisticTest, test_Init) { NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(0, allocStatistic_->Init()); @@ -102,8 +105,9 @@ TEST_F(AllocStatisticTest, test_PeriodicPersist_CalculateSegmentAlloc) { NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(0, allocStatistic_->Init()); diff --git a/test/mds/nameserver2/clean_core_test.cpp b/test/mds/nameserver2/clean_core_test.cpp index 47cefd8595..b428c2e75a 100644 --- a/test/mds/nameserver2/clean_core_test.cpp +++ b/test/mds/nameserver2/clean_core_test.cpp @@ -28,33 +28,56 @@ #include "test/mds/mock/mock_topology.h" #include "src/mds/chunkserverclient/copyset_client.h" #include "test/mds/mock/mock_alloc_statistic.h" +#include "test/mds/mock/mock_chunkserverclient.h" using ::testing::_; using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::DoAll; using curve::mds::topology::MockTopology; using ::curve::mds::chunkserverclient::ChunkServerClientOption; +using ::curve::mds::chunkserverclient::MockChunkServerClient; namespace curve { namespace mds { -TEST(CleanCore, testcleansnapshotfile) { - auto storage = std::make_shared(); - auto topology = std::make_shared(); - ChunkServerClientOption option; - auto channelPool = std::make_shared(); - auto client = std::make_shared(topology, - option, channelPool); - auto allocStatistic = std::make_shared(); - auto cleanCore = std::make_shared(storage, - client, allocStatistic); +class CleanCoreTest : public testing::Test { + public: + void SetUp() override { + storage_ = std::make_shared(); + topology_ = std::make_shared(); + channelPool_ = std::make_shared(); + client_ = + std::make_shared(topology_, option_, channelPool_); + allocStatistic_ = std::make_shared(); + cleanCore_ = + std::make_shared(storage_, client_, allocStatistic_); + + csClient_ = std::make_shared( + topology_, option_, channelPool_); + } + + void TearDown() override {} + protected: + std::shared_ptr storage_; + std::shared_ptr topology_; + ChunkServerClientOption option_; + std::shared_ptr channelPool_; + std::shared_ptr client_; + std::shared_ptr allocStatistic_; + std::shared_ptr cleanCore_; + std::shared_ptr csClient_; +}; + +TEST_F(CleanCoreTest, testcleansnapshotfile) { { // segment size = 0 FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(0); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::KInternalError); } @@ -62,11 +85,11 @@ TEST(CleanCore, testcleansnapshotfile) { // delete ok (no, segment) uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -74,7 +97,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -84,24 +107,24 @@ TEST(CleanCore, testcleansnapshotfile) { // all ok , but do DeleteFile namespace meta error uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kSnapshotFileDeleteError); } { // get segment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .Times(1) .WillOnce(Return(StoreStatus::InternalError)); @@ -109,7 +132,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kSnapshotFileDeleteError); } { @@ -118,12 +141,12 @@ TEST(CleanCore, testcleansnapshotfile) { uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; uint64_t expectParentID = 101; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, + EXPECT_CALL(*storage_, GetSegment(expectParentID, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -132,7 +155,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_segmentsize(DefaultSegmentSize); cleanFile.set_parentid(expectParentID); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -146,11 +169,11 @@ TEST(CleanCore, testcleansnapshotfile) { // get segment ok, DeleteSnapShotChunk OK uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::OK)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -158,7 +181,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -166,24 +189,14 @@ TEST(CleanCore, testcleansnapshotfile) { } } -TEST(CleanCore, testcleanfile) { - auto storage = std::make_shared(); - auto topology = std::make_shared(); - ChunkServerClientOption option; - auto channelPool = std::make_shared(); - auto client = std::make_shared(topology, - option, channelPool); - auto allocStatistic = std::make_shared(); - auto cleanCore = std::make_shared(storage, - client, allocStatistic); - +TEST_F(CleanCoreTest, testcleanfile) { { // segmentsize = 0 FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(0); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::KInternalError); } @@ -191,11 +204,11 @@ TEST(CleanCore, testcleanfile) { // delete ok (no, segment) uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteFile(_, _)) + EXPECT_CALL(*storage_, DeleteFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -203,7 +216,7 @@ TEST(CleanCore, testcleanfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -214,25 +227,25 @@ TEST(CleanCore, testcleanfile) { // all ok , but do DeleteFile namespace meta error uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteFile(_, _)) + EXPECT_CALL(*storage_, DeleteFile(_, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } { // get segment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .Times(1) .WillOnce(Return(StoreStatus::InternalError)); @@ -240,7 +253,7 @@ TEST(CleanCore, testcleanfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } @@ -249,20 +262,120 @@ TEST(CleanCore, testcleanfile) { } { // get segment ok, DeleteSnapShotChunk ok, DeleteSegment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .WillOnce(Return(StoreStatus::OK)); - EXPECT_CALL(*storage, DeleteSegment(_, _, _)) + EXPECT_CALL(*storage_, DeleteSegment(_, _, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } } + +TEST_F(CleanCoreTest, TestCleanDiscardSegment) { + const std::string fakeKey = "fakekey"; + const int kDefaultChunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("/test_file"); + fileInfo.set_id(1234); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_length(kMiniFileLength); + + PageFileSegment segment; + segment.set_logicalpoolid(1); + segment.set_segmentsize(DefaultSegmentSize); + segment.set_chunksize(kDefaultChunkSize); + segment.set_startoffset(0); + + for (int i = 0; i < DefaultSegmentSize / kDefaultChunkSize; ++i) { + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(i); + chunk->set_chunkid(i); + } + + DiscardSegmentInfo discardSegmentInfo; + discardSegmentInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardSegmentInfo.set_allocated_pagefilesegment( + new PageFileSegment(segment)); + + // CopysetClient DeleteChunk failed + { + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .WillOnce(Return(false)); + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _)) + .Times(0); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(0); + TaskProgress progress; + ASSERT_EQ(StatusCode::KInternalError, + cleanCore_->CleanDiscardSegment(fakeKey, discardSegmentInfo, + &progress)); + ASSERT_EQ(TaskStatus::FAILED, progress.GetStatus()); + } + + // NameServerStorage CleanDiscardSegment failed + { + client_->SetChunkServerClient(csClient_); + + CopySetInfo copyset; + + copyset.SetLeader(1); + + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .Times(segment.chunks_size()) + .WillRepeatedly( + DoAll(SetArgPointee<1>(copyset), Return(true))); + EXPECT_CALL(*csClient_, DeleteChunk(_, _, _, _, _)) + .Times(segment.chunks_size()) + .WillRepeatedly(Return(kMdsSuccess)); + + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::InternalError)); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(0); + + TaskProgress progress; + ASSERT_EQ(StatusCode::KInternalError, + cleanCore_->CleanDiscardSegment(fakeKey, discardSegmentInfo, + &progress)); + ASSERT_EQ(TaskStatus::FAILED, progress.GetStatus()); + } + + // ok + { + client_->SetChunkServerClient(csClient_); + + CopySetInfo copyset; + + copyset.SetLeader(1); + + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .Times(segment.chunks_size()) + .WillRepeatedly( + DoAll(SetArgPointee<1>(copyset), Return(true))); + EXPECT_CALL(*csClient_, DeleteChunk(_, _, _, _, _)) + .Times(segment.chunks_size()) + .WillRepeatedly(Return(kMdsSuccess)); + + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::OK)); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(1); + + TaskProgress progress; + ASSERT_EQ(StatusCode::kOK, cleanCore_->CleanDiscardSegment( + fakeKey, discardSegmentInfo, &progress)); + ASSERT_EQ(100, progress.GetProgress()); + ASSERT_EQ(TaskStatus::SUCCESS, progress.GetStatus()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/nameserver2/clean_discard_segment_task_test.cpp b/test/mds/nameserver2/clean_discard_segment_task_test.cpp new file mode 100644 index 0000000000..46f5bbcc94 --- /dev/null +++ b/test/mds/nameserver2/clean_discard_segment_task_test.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Dec 22 14:25:52 CST 2020 + */ + +#include +#include +#include + +#include "test/mds/mock/mock_alloc_statistic.h" +#include "test/mds/mock/mock_chunkserverclient.h" +#include "test/mds/mock/mock_topology.h" +#include "test/mds/nameserver2/mock/mock_clean_manager.h" +#include "test/mds/nameserver2/mock/mock_namespace_storage.h" + +namespace curve { +namespace mds { + +using ::curve::mds::chunkserverclient::ChunkServerClientOption; +using ::curve::mds::chunkserverclient::MockChunkServerClient; +using curve::mds::topology::MockTopology; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SetArgPointee; + +class CleanDiscardSegmentTaskTest : public ::testing::Test { + public: + void SetUp() override { + storage_ = std::make_shared(); + topology_ = std::make_shared(); + channelPool_ = std::make_shared(); + client_ = + std::make_shared(topology_, option_, channelPool_); + allocStatistic_ = std::make_shared(); + cleanCore_ = + std::make_shared(storage_, client_, allocStatistic_); + + csClient_ = std::make_shared(topology_, option_, + channelPool_); + + cleanManager_ = std::make_shared(); + } + + void TearDown() override {} + + protected: + std::shared_ptr storage_; + std::shared_ptr topology_; + ChunkServerClientOption option_; + std::shared_ptr channelPool_; + std::shared_ptr client_; + std::shared_ptr allocStatistic_; + std::shared_ptr cleanCore_; + std::shared_ptr csClient_; + std::shared_ptr cleanManager_; +}; + +TEST_F(CleanDiscardSegmentTaskTest, CommonTest) { + CleanDiscardSegmentTask task(cleanManager_, storage_, 50); + std::map discardSegments; + discardSegments.emplace("hello", DiscardSegmentInfo()); + + EXPECT_CALL(*storage_, ListDiscardSegment(_)) + .WillRepeatedly( + DoAll(SetArgPointee<0>(discardSegments), + Return(StoreStatus::OK))); + + EXPECT_CALL(*cleanManager_, SubmitCleanDiscardSegmentJob(_, _)) + .WillRepeatedly(Return(true)); + + task.Start(); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_NO_THROW(task.Stop()); +} + +TEST_F(CleanDiscardSegmentTaskTest, TestListDiscardSegmentFailed) { + CleanDiscardSegmentTask task(cleanManager_, storage_, 50); + std::map discardSegments; + discardSegments.emplace("hello", DiscardSegmentInfo()); + + EXPECT_CALL(*storage_, ListDiscardSegment(_)) + .WillRepeatedly( + Return(StoreStatus::InternalError)); + + EXPECT_CALL(*cleanManager_, SubmitCleanDiscardSegmentJob(_, _)) + .Times(0); + + task.Start(); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_NO_THROW(task.Stop()); +} + +TEST_F(CleanDiscardSegmentTaskTest, TestSubmitJobFailed) { + CleanDiscardSegmentTask task(cleanManager_, storage_, 50); + std::map discardSegments; + discardSegments.emplace("hello", DiscardSegmentInfo()); + + EXPECT_CALL(*storage_, ListDiscardSegment(_)) + .WillRepeatedly( + DoAll(SetArgPointee<0>(discardSegments), + Return(StoreStatus::OK))); + + EXPECT_CALL(*cleanManager_, SubmitCleanDiscardSegmentJob(_, _)) + .WillRepeatedly(Return(false)); + + task.Start(); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_NO_THROW(task.Stop()); +} + +} // namespace mds +} // namespace curve diff --git a/test/mds/nameserver2/curvefs_test.cpp b/test/mds/nameserver2/curvefs_test.cpp index e7f088733b..a2abfc4334 100644 --- a/test/mds/nameserver2/curvefs_test.cpp +++ b/test/mds/nameserver2/curvefs_test.cpp @@ -1903,6 +1903,218 @@ TEST_F(CurveFSTest, testGetOrAllocateSegment) { } } +TEST_F(CurveFSTest, TestDeAllocateSegment) { + const std::string filename = "/TestDeAllocateSegment"; + const uint64_t offset = 1ull * 1024 * 1024 * 1024; + + // GetFileInfo failed + { + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file type not support + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_DIRECTORY); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment offset wrong + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(100 * kGB); + fileInfo.set_segmentsize(1 * kGB); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, 100 * kGB)); + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, 1 * kGB + 1)); + } + + // file is being cloned + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kNotSupported, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment not exists + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce(Return(StoreStatus::KeyNotExist)); + + ASSERT_EQ(StatusCode::kSegmentNotAllocated, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment offset not equal to argument + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(0ull); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // list snapshot failed + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file under snapshot + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + snapshotFiles.push_back(FileInfo()); + snapshotFiles.push_back(FileInfo()); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kFileUnderSnapShot, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // storage discard segment failed + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, DiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // storage discard segment success + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, DiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(StatusCode::kOK, + curvefs_->DeAllocateSegment(filename, offset)); + } +} + TEST_F(CurveFSTest, testCreateSnapshotFile) { { // test client time not expired diff --git a/test/mds/nameserver2/fakes.h b/test/mds/nameserver2/fakes.h index a1345e17e5..e3f7e8d2af 100644 --- a/test/mds/nameserver2/fakes.h +++ b/test/mds/nameserver2/fakes.h @@ -411,6 +411,55 @@ class FakeNameServerStorage : public NameServerStorage { return StoreStatus::OK; } + StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) override { + std::lock_guard guard(lock_); + std::string segmentKey = NameSpaceStorageCodec::EncodeSegmentStoreKey( + fileInfo.id(), segment.startoffset()); + std::string cleanSegmentKey = + NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey( + fileInfo.id(), segment.startoffset()); + + std::string encodeSegment; + if (!NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)) { + return StoreStatus::InternalError; + } + + std::string encodeDiscardSegment; + DiscardSegmentInfo discardInfo; + discardInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardInfo.set_allocated_pagefilesegment(new PageFileSegment(segment)); + if (!NameSpaceStorageCodec::EncodeDiscardSegment( + discardInfo, &encodeDiscardSegment)) { + return StoreStatus::InternalError; + } + + if (memKvMap_.count(segmentKey) == 0) { + return StoreStatus::KeyNotExist; + } + + memKvMap_.erase(segmentKey); + memKvMap_.emplace(cleanSegmentKey, encodeDiscardSegment); + + return StoreStatus::OK; + } + + StoreStatus CleanDiscardSegment(const std::string& key, + int64_t* revision) override { + std::lock_guard guard(lock_); + if (memKvMap_.count(key) == 0) { + return StoreStatus::KeyNotExist; + } + + memKvMap_.erase(key); + return StoreStatus::OK; + } + + StoreStatus ListDiscardSegment( + std::map* out) override { + return StoreStatus::OK; + } + private: std::mutex lock_; std::map memKvMap_; diff --git a/test/mds/nameserver2/mock/mock_clean_manager.h b/test/mds/nameserver2/mock/mock_clean_manager.h index 1680bd3d56..0439d6d9a8 100644 --- a/test/mds/nameserver2/mock/mock_clean_manager.h +++ b/test/mds/nameserver2/mock/mock_clean_manager.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "src/mds/nameserver2/clean_manager.h" #include "src/mds/nameserver2/async_delete_snapshot_entity.h" @@ -39,6 +40,8 @@ class MockCleanManager: public CleanManagerInterface { std::shared_ptr)); MOCK_METHOD1(GetTask, std::shared_ptr(TaskIDType id)); MOCK_METHOD1(SubmitDeleteCommonFileJob, bool(const FileInfo&)); + MOCK_METHOD2(SubmitCleanDiscardSegmentJob, + bool(const std::string&, const DiscardSegmentInfo&)); }; } // namespace mds diff --git a/test/mds/nameserver2/mock/mock_namespace_storage.h b/test/mds/nameserver2/mock/mock_namespace_storage.h index 2f3da1759a..d6191dba6a 100644 --- a/test/mds/nameserver2/mock/mock_namespace_storage.h +++ b/test/mds/nameserver2/mock/mock_namespace_storage.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "src/mds/nameserver2/namespace_storage.h" namespace curve { @@ -84,6 +85,13 @@ class MockNameServerStorage : public NameServerStorage { StoreStatus(std::vector *snapShotFiles)); MOCK_METHOD2(ListSegment, StoreStatus(InodeID, std::vector*)); + + MOCK_METHOD2(DiscardSegment, + StoreStatus(const FileInfo&, const PageFileSegment&)); + MOCK_METHOD2(CleanDiscardSegment, + StoreStatus(const std::string&, int64_t*)); + MOCK_METHOD1(ListDiscardSegment, + StoreStatus(std::map*)); }; } // namespace mds diff --git a/test/mds/nameserver2/namespace_service_test.cpp b/test/mds/nameserver2/namespace_service_test.cpp index 93b0c5fcdd..e60e662d5f 100644 --- a/test/mds/nameserver2/namespace_service_test.cpp +++ b/test/mds/nameserver2/namespace_service_test.cpp @@ -2012,6 +2012,125 @@ TEST_F(NameSpaceServiceTest, FindFileMountPointTest) { server.Join(); } +TEST_F(NameSpaceServiceTest, TestDeAllocateSegment) { + brpc::Server server; + + NameSpaceService nameSpaceSerivce(new FileLockManager(13)); + + ASSERT_EQ(0, server.AddService(&nameSpaceSerivce, + brpc::SERVER_DOESNT_OWN_SERVICE)); + + brpc::ServerOptions opt; + opt.idle_timeout_sec = -1; + ASSERT_EQ(0, server.Start("127.0.0.1", {8900, 8999}, &opt)); + + // init channel + brpc::Channel channel; + ASSERT_EQ(channel.Init(server.listen_address(), nullptr), 0); + + CurveFSService_Stub stub(&channel); + brpc::Controller cntl; + uint64_t fileLength = 100 * kGB; + std::string filename = "/TestDeAllocateSegment"; + std::string owner = "curve"; + + // create file and allocate segment + { + CreateFileRequest createRequest; + CreateFileResponse createResponse; + createRequest.set_filename(filename); + createRequest.set_owner(owner); + createRequest.set_date(TimeUtility::GetTimeofDayUs()); + createRequest.set_filetype(INODE_PAGEFILE); + createRequest.set_filelength(fileLength); + + cntl.set_log_id(1); + stub.CreateFile(&cntl, &createRequest, &createResponse, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kOK, createResponse.statuscode()); + + // allocate segment + cntl.Reset(); + GetOrAllocateSegmentRequest allocateRequest; + GetOrAllocateSegmentResponse allocateResponse; + + allocateRequest.set_filename(filename); + allocateRequest.set_offset(50ull * kGB); + allocateRequest.set_allocateifnotexist(true); + allocateRequest.set_owner(owner); + allocateRequest.set_date(TimeUtility::GetTimeofDayUs()); + + stub.GetOrAllocateSegment(&cntl, &allocateRequest, &allocateResponse, + nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kOK, allocateResponse.statuscode()); + } + + // 1. filename not valid + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename + "/"); + request.set_offset(0); + request.set_owner("curve"); + request.set_date(TimeUtility::GetTimeofDayUs()); + + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kParaError, response.statuscode()); + } + + // 2. segment not allocated + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename); + request.set_offset(0); + request.set_owner(owner); + request.set_date(TimeUtility::GetTimeofDayUs()); + + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kSegmentNotAllocated, response.statuscode()); + } + + // 3. deallocate success + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename); + request.set_offset(50ull * kGB); + request.set_owner(owner); + request.set_date(TimeUtility::GetTimeofDayUs()); + + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kOK, response.statuscode()); + } + + // 4. retry rpc + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename); + request.set_offset(50ull * kGB); + request.set_owner(owner); + request.set_date(TimeUtility::GetTimeofDayUs()); + + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kSegmentNotAllocated, response.statuscode()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/nameserver2/namespace_storage_test.cpp b/test/mds/nameserver2/namespace_storage_test.cpp index 968a57d71c..99f6175360 100644 --- a/test/mds/nameserver2/namespace_storage_test.cpp +++ b/test/mds/nameserver2/namespace_storage_test.cpp @@ -34,6 +34,7 @@ using ::testing::Return; using ::testing::AtLeast; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace mds { @@ -298,7 +299,7 @@ TEST_F(TestNameServerStorageImp, test_MoveFileToRecycle) { TEST_F(TestNameServerStorageImp, test_ListFile) { // 1. list err std::vector listRes; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(StoreStatus::InternalError, storage_->ListFile(0, 0, &listRes)); @@ -309,7 +310,7 @@ TEST_F(TestNameServerStorageImp, test_ListFile) { GetFileInfoForTest(&fileinfo); ASSERT_TRUE(NameSpaceStorageCodec::EncodeFileInfo(fileinfo, &encodeFileinfo)); - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(DoAll( SetArgPointee<2>(std::vector{encodeFileinfo}), Return(EtcdErrCode::EtcdOK))); @@ -322,7 +323,7 @@ TEST_F(TestNameServerStorageImp, test_ListFile) { TEST_F(TestNameServerStorageImp, test_ListSnapshotFile) { // 1. list err std::vector listRes; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ( StoreStatus::InternalError, storage_->ListSnapshotFile(1, 2, &listRes)); @@ -339,10 +340,11 @@ TEST_F(TestNameServerStorageImp, test_ListSnapshotFile) { std::string endStoreKey = NameSpaceStorageCodec::EncodeSnapShotFileStoreKey(2, ""); - EXPECT_CALL(*client_, List(startStoreKey, endStoreKey, _)) - .WillOnce(DoAll( - SetArgPointee<2>(std::vector{encodeFileinfo}), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*client_, List(startStoreKey, endStoreKey, + Matcher*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(std::vector{encodeFileinfo}), + Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(StoreStatus::OK, storage_->ListSnapshotFile(1, 2, &listRes)); ASSERT_EQ(1, listRes.size()); ASSERT_EQ(fileinfo.filename(), listRes[0].filename()); @@ -420,7 +422,7 @@ TEST_F(TestNameServerStorageImp, test_Snapshotfile) { TEST_F(TestNameServerStorageImp, test_ListSegment) { // 1. list err std::vector segments; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(StoreStatus::InternalError, storage_->ListSegment(0, &segments)); @@ -430,7 +432,7 @@ TEST_F(TestNameServerStorageImp, test_ListSegment) { PageFileSegment segment; GetPageFileSegmentForTest(&key, &segment); ASSERT_TRUE(NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(DoAll( SetArgPointee<2>(std::vector{encodeSegment}), Return(EtcdErrCode::EtcdOK))); @@ -439,5 +441,137 @@ TEST_F(TestNameServerStorageImp, test_ListSegment) { ASSERT_EQ(segment.DebugString(), segments[0].DebugString()); } +TEST_F(TestNameServerStorageImp, test_DiscardSegment) { + const uint32_t chunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("test_DiscardSegment"); + + PageFileSegment segment; + segment.set_logicalpoolid(0); + segment.set_segmentsize(chunkSize); + segment.set_chunksize(chunkSize); + segment.set_startoffset(0); + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(1); + chunk->set_chunkid(2); + + // transaction failed + { + EXPECT_CALL(*client_, TxnN(_)) + .WillOnce(Return(EtcdErrCode::EtcdTxnUnkownOp)); + + ASSERT_EQ(StoreStatus::InternalError, + storage_->DiscardSegment(fileInfo, segment)); + } + + // ok + { + EXPECT_CALL(*client_, TxnN(_)).WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*cache_, Remove(_)).Times(1); + + ASSERT_EQ(StoreStatus::OK, storage_->DiscardSegment(fileInfo, segment)); + } +} + +TEST_F(TestNameServerStorageImp, test_CleanDisardSegment) { + // delete failed + { + EXPECT_CALL(*client_, DeleteRewithRevision(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + + std::string key = "fakekey"; + int64_t revision; + ASSERT_EQ(StoreStatus::InternalError, + storage_->CleanDiscardSegment(key, &revision)); + } + + // delete ok + { + EXPECT_CALL(*client_, DeleteRewithRevision(_, _)) + .WillOnce( + DoAll(SetArgPointee<1>(100), Return(EtcdErrCode::EtcdOK))); + + std::string key = "fakekey"; + int64_t revision; + ASSERT_EQ(StoreStatus::OK, + storage_->CleanDiscardSegment(key, &revision)); + ASSERT_EQ(100, revision); + } +} + +TEST_F(TestNameServerStorageImp, test_ListDiscardSegment) { + // list failed + { + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + + std::map out; + ASSERT_EQ(StoreStatus::InternalError, + storage_->ListDiscardSegment(&out)); + } + + // decode failed + { + std::vector> kvs{ + {"hello", "world"}}; + + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(kvs), Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(StoreStatus::InternalError, + storage_->ListDiscardSegment(&out)); + } + + // ok + { + const uint32_t chunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("test_DiscardSegment"); + + PageFileSegment segment; + segment.set_logicalpoolid(0); + segment.set_segmentsize(chunkSize); + segment.set_chunksize(chunkSize); + segment.set_startoffset(0); + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(1); + chunk->set_chunkid(2); + + DiscardSegmentInfo discardSegmentInfo; + discardSegmentInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardSegmentInfo.set_allocated_pagefilesegment( + new PageFileSegment(segment)); + + std::string encodeDiscardSegmentInfo; + ASSERT_TRUE(NameSpaceStorageCodec::EncodeDiscardSegment( + discardSegmentInfo, &encodeDiscardSegmentInfo)); + + std::vector> kvs{ + {"hello", encodeDiscardSegmentInfo}}; + + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(kvs), Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(StoreStatus::OK, storage_->ListDiscardSegment(&out)); + + ASSERT_EQ(discardSegmentInfo.DebugString(), out["hello"].DebugString()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/topology/mock_topology.h b/test/mds/topology/mock_topology.h index a9dbab5fb9..362de4d07c 100644 --- a/test/mds/topology/mock_topology.h +++ b/test/mds/topology/mock_topology.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "proto/topology.pb.h" #include "src/mds/topology/topology_service_manager.h" @@ -327,6 +328,8 @@ class MockKVStorageClient : public KVStorageClient { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/mds/topology/test_topology_storage_etcd.cpp b/test/mds/topology/test_topology_storage_etcd.cpp index 8155a961e8..32bf8f6054 100644 --- a/test/mds/topology/test_topology_storage_etcd.cpp +++ b/test/mds/topology/test_topology_storage_etcd.cpp @@ -36,6 +36,7 @@ using ::testing::AllOf; using ::testing::SetArgPointee; using ::testing::Invoke; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace mds { @@ -77,9 +78,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success) { ASSERT_TRUE(codec_->EncodeLogicalPoolData(data, &value)); std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -92,7 +93,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success) { } TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success_ListEtcdEmpty) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); std::unordered_map logicalPoolMap; @@ -105,7 +107,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success_ListEtcdEmpty) { } TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_ListEtcdFail) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); std::unordered_map logicalPoolMap; @@ -120,9 +123,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_ListEtcdFail) { TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -147,9 +150,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -166,9 +169,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -182,7 +185,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success) { } TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_listEtcdEmpty) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); std::unordered_map physicalPoolMap; @@ -198,9 +202,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -219,9 +223,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -239,9 +243,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -256,9 +260,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success) { TEST_F(TestTopologyStorageEtcd, test_LoadZone_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -273,9 +278,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadZone_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -294,9 +299,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -315,9 +320,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -332,9 +337,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success) { TEST_F(TestTopologyStorageEtcd, test_LoadServer_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -349,9 +355,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadServer_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -371,9 +377,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -399,9 +405,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -417,9 +423,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success) { TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -434,9 +441,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -463,9 +470,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_IdDuplcated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -486,9 +493,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; @@ -505,9 +512,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success) { TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::map copySetMap; std::map copySetIdMaxMap; @@ -522,9 +530,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; @@ -546,9 +554,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; diff --git a/test/snapshotcloneserver/mock_snapshot_server.h b/test/snapshotcloneserver/mock_snapshot_server.h index 4d2272592c..35cb3a6632 100644 --- a/test/snapshotcloneserver/mock_snapshot_server.h +++ b/test/snapshotcloneserver/mock_snapshot_server.h @@ -29,6 +29,7 @@ #include #include #include +#include #include "src/snapshotcloneserver/snapshot/snapshot_core.h" #include "src/snapshotcloneserver/clone/clone_core.h" @@ -418,6 +419,8 @@ class MockKVStorageClient : public KVStorageClient { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp index 2dbf55db45..40d331035c 100644 --- a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp +++ b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp @@ -47,6 +47,7 @@ using ::testing::AllOf; using ::testing::SetArgPointee; using ::testing::Invoke; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace snapshotcloneserver { @@ -567,7 +568,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector cloneOut; cloneOut.push_back(cloneValue); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(DoAll(SetArgPointee<2>(cloneOut), @@ -579,7 +580,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, TEST_F(TestSnapshotCloneMetaStoreEtcd, TestInitListSnapshotFail) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(Return(EtcdErrCode::EtcdUnknown)); int ret = metaStore_->Init(); @@ -597,7 +598,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out; out.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); @@ -611,7 +612,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out; out.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))); @@ -632,7 +633,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out2; out2.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(DoAll(SetArgPointee<2>(out2), diff --git a/thirdparties/etcdclient/Makefile b/thirdparties/etcdclient/Makefile index 1edc5d82c1..0b82d3b0a9 100644 --- a/thirdparties/etcdclient/Makefile +++ b/thirdparties/etcdclient/Makefile @@ -22,12 +22,12 @@ all: intall-go install-etcdclient libetcdclient intall-go: mkdir -p $(pwd)/tmp - cd $(pwd)/tmp && wget -N https://dl.google.com/go/go1.12.8.linux-amd64.tar.gz + cd $(pwd)/tmp && wget -N https://curve-build.nos-eastchina1.126.net/go1.12.8.linux-amd64.tar.gz cd $(pwd)/tmp && tar zxvf go1.12.8.linux-amd64.tar.gz install-etcdclient: mkdir -p $(pwd)/tmp/gosrc/src/go.etcd.io - cd $(pwd)/tmp/gosrc/src/go.etcd.io && git clone --branch v3.4.0 https://github.com/etcd-io/etcd + cd $(pwd)/tmp/gosrc/src/go.etcd.io && git clone --branch v3.4.0 https://gitee.com/mirrors/etcd cd $(pwd)/tmp/gosrc/src/go.etcd.io/etcd && cp $(pwd)/expose-session-for-election.patch . && patch -p1 < expose-session-for-election.patch vendorpath := $(pwd)/tmp/gosrc/src/go.etcd.io/etcd/vendor