From 7f129267f877fcdf4d99e57c010f6cc078aedb9b Mon Sep 17 00:00:00 2001 From: hzchenwei7 Date: Wed, 27 May 2020 16:29:37 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BF=AB=E7=85=A7=E5=85=8B=E9=9A=86=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E9=95=9C=E5=83=8F=E5=8F=8A=E5=BF=AB=E7=85=A7=E5=88=A0?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I622e63a2fbd52c62fce33ce4eb3ad82b974bd624 快照克隆版本镜像及快照删除2 fix review feedback --- conf/mds.conf | 6 + conf/snapshot_clone_server.conf | 4 + .../roles/generate_config/defaults/main.yml | 1 + .../generate_config/templates/mds.conf.j2 | 5 + proto/nameserver2.proto | 4 + src/common/BUILD | 16 +- src/{snapshotcloneserver => }/common/define.h | 6 +- src/common/snapshotclone/BUILD | 46 +++ .../snapshotclone/snapshotclone_define.cpp} | 6 +- .../snapshotclone/snapshotclone_define.h | 160 ++++++++ src/mds/nameserver2/BUILD | 1 + src/mds/nameserver2/curvefs.cpp | 75 +++- src/mds/nameserver2/curvefs.h | 19 +- src/mds/server/mds.cpp | 22 +- src/mds/server/mds.h | 9 +- src/mds/snapshotcloneclient/BUILD | 38 ++ .../snapshotclone_client.cpp | 147 +++++++ .../snapshotclone_client.h | 85 ++++ src/snapshotcloneserver/BUILD | 2 + src/snapshotcloneserver/clone/clone_closure.h | 2 +- src/snapshotcloneserver/clone/clone_core.cpp | 87 +++++ src/snapshotcloneserver/clone/clone_core.h | 25 +- .../clone/clone_reference.h | 2 +- .../clone/clone_service_manager.cpp | 145 ++++++- .../clone/clone_service_manager.h | 70 +++- src/snapshotcloneserver/clone/clone_task.h | 2 +- .../clone/clone_task_manager.cpp | 2 +- .../clone/clone_task_manager.h | 6 +- src/snapshotcloneserver/common/config.h | 4 + .../common/curvefs_client.h | 2 +- .../common/snapshot_reference.h | 2 +- .../common/snapshotclone_info.h | 2 +- .../common/snapshotclone_meta_store.h | 2 +- .../common/snapshotclone_meta_store_etcd.cpp | 15 +- src/snapshotcloneserver/common/task.h | 2 +- src/snapshotcloneserver/common/task_tracker.h | 2 +- .../snapshot/snapshot_core.cpp | 2 +- .../snapshot/snapshot_core.h | 2 +- .../snapshot/snapshot_service_manager.h | 2 +- .../snapshot/snapshot_task.h | 2 +- .../snapshot/snapshot_task_manager.cpp | 2 +- .../snapshot/snapshot_task_manager.h | 2 +- .../snapshotclone_server.cpp | 12 +- .../snapshotclone_server.h | 3 +- .../snapshotclone_service.cpp | 71 +++- .../snapshotclone_service.h | 2 + test/chunkserver/heartbeat_test_common.h | 2 +- test/integration/heartbeat/common.h | 4 +- .../fake_snapshotclone_meta_store.cpp | 4 +- .../snapshotcloneserver_common_test.cpp | 65 +++- .../snapshotcloneserver_concurrent_test.cpp | 5 +- .../snapshotcloneserver_exception_test.cpp | 2 + .../snapshotcloneserver_module.cpp | 5 +- .../snapshotcloneserver_module.h | 2 +- .../snapshotcloneserver_recover_test.cpp | 5 +- .../snapshotcloneserver_test.cpp | 3 + test/mds/mock/BUILD | 1 + test/mds/mock/mock_snapshotcloneserver.h | 47 +++ test/mds/nameserver2/curvefs_test.cpp | 253 +++++++++++- test/mds/nameserver2/file_record_test.cpp | 4 +- test/mds/nameserver2/mock/BUILD | 1 + .../mock/mock_snapshotclone_client.h | 45 +++ .../nameserver2/namespace_service_test.cpp | 3 +- test/mds/snapshotcloneclient/BUILD | 33 ++ .../test_snapshotclone_client.cpp | 358 +++++++++++++++++ .../mock_snapshot_server.h | 21 +- test/snapshotcloneserver/test_client.conf | 32 ++ test/snapshotcloneserver/test_clone_core.cpp | 111 +++++- .../test_clone_service_manager.cpp | 363 +++++++++++++++++- .../test_snapshot_core.cpp | 2 +- .../test_snapshot_service_manager.cpp | 4 +- .../test_snapshotclone_meta_store_etcd.cpp | 20 +- .../test_snapshotclone_service.cpp | 255 ++++++++++++ 73 files changed, 2694 insertions(+), 80 deletions(-) rename src/{snapshotcloneserver => }/common/define.h (96%) create mode 100644 src/common/snapshotclone/BUILD rename src/{snapshotcloneserver/common/define.cpp => common/snapshotclone/snapshotclone_define.cpp} (93%) create mode 100644 src/common/snapshotclone/snapshotclone_define.h create mode 100644 src/mds/snapshotcloneclient/BUILD create mode 100644 src/mds/snapshotcloneclient/snapshotclone_client.cpp create mode 100644 src/mds/snapshotcloneclient/snapshotclone_client.h create mode 100644 test/mds/mock/mock_snapshotcloneserver.h create mode 100644 test/mds/nameserver2/mock/mock_snapshotclone_client.h create mode 100644 test/mds/snapshotcloneclient/BUILD create mode 100644 test/mds/snapshotcloneclient/test_snapshotclone_client.cpp create mode 100644 test/snapshotcloneserver/test_client.conf diff --git a/conf/mds.conf b/conf/mds.conf index b7007b30e4..07c7f673f0 100644 --- a/conf/mds.conf +++ b/conf/mds.conf @@ -180,6 +180,12 @@ mds.chunkserverclient.updateLeaderRetryTimes=5 # 从copyset的每个chunkserver getleader的每一轮的间隔，需大于raft选主的时间 mds.chunkserverclient.updateLeaderRetryIntervalMs=5000 +# +# snapshotclone config +# +# snapshot clone server 地址 +mds.snapshotcloneclient.addr=127.0.0.1:5555 + # # common options # diff --git a/conf/snapshot_clone_server.conf b/conf/snapshot_clone_server.conf index b94e0b6c54..ff62d0a290 100644 --- a/conf/snapshot_clone_server.conf +++ b/conf/snapshot_clone_server.conf @@ -64,6 +64,10 @@ server.cloneTempDir=/clone server.createCloneChunkConcurrency=64 # RecoverChunk同时进行的异步请求数量 server.recoverChunkConcurrency=64 +# CloneServiceManager引用计数后台扫描每条记录间隔 +server.backEndReferenceRecordScanIntervalMs=500 +# CloneServiceManager引用计数后台扫描每轮记录间隔 +server.backEndReferenceFuncScanIntervalMs=3600000 # # etcd相关配置 diff --git a/curve-ansible/roles/generate_config/defaults/main.yml b/curve-ansible/roles/generate_config/defaults/main.yml index a81f7e7b3f..1b5138bbe8 100644 --- a/curve-ansible/roles/generate_config/defaults/main.yml +++ b/curve-ansible/roles/generate_config/defaults/main.yml @@ -70,6 +70,7 @@ mds_chunkserverclient_rpc_retry_times: 5 mds_chunkserverclient_rpc_retry_interval_ms: 500 mds_chunkserverclient_update_leader_retry_times: 5 mds_chunkserverclient_update_leader_retry_interval_ms: 5000 +mds_snapshotcloneclient_addr: 127.0.0.1:5555 mds_common_log_dir: ./ # chunkserver配置默认值 diff --git a/curve-ansible/roles/generate_config/templates/mds.conf.j2 b/curve-ansible/roles/generate_config/templates/mds.conf.j2 index d548938fc1..e23d619b60 100644 --- a/curve-ansible/roles/generate_config/templates/mds.conf.j2 +++ b/curve-ansible/roles/generate_config/templates/mds.conf.j2 @@ -188,6 +188,11 @@ mds.chunkserverclient.updateLeaderRetryTimes={{ mds_chunkserverclient_update_lea # 从copyset的每个chunkserver getleader的每一轮的间隔，需大于raft选主的时间 mds.chunkserverclient.updateLeaderRetryIntervalMs={{ mds_chunkserverclient_update_leader_retry_interval_ms }} +# snapshotclone config +# +# snapshot clone server 地址 +mds.snapshotcloneclient.addr={{ mds_snapshotcloneclient_addr }} + # # common options # diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto index 991808a936..53e6ffbf2b 100644 --- a/proto/nameserver2.proto +++ b/proto/nameserver2.proto @@ -130,6 +130,10 @@ enum StatusCode { kClientVersionNotMatch = 134; // snapshot功能禁用中 kSnapshotFrozen = 135; + // 快照克隆服务连不上 + kSnapshotCloneConnectFail = 136; + // 快照克隆服务未初始化 + kSnapshotCloneServerNotInit = 137; // 元数据存储错误 kStorageError = 501; diff --git a/src/common/BUILD b/src/common/BUILD index 7ddaefe973..588634bed2 100644 --- a/src/common/BUILD +++ b/src/common/BUILD @@ -38,7 +38,9 @@ cc_library( srcs = glob([ "*.h", "*.cpp", - ],exclude = ["authenticator.*", "s3_adapter.*"] + ],exclude = ["authenticator.*", + "s3_adapter.*", + "snapshotclone_define.*"] ), copts = COPTS, visibility = ["//visibility:public"], @@ -91,3 +93,15 @@ cc_library( "@aws", ], ) + +cc_library( + name = "curve_snapshotclone", + srcs = glob([ + "snapshotclone_define.*", + ]), + copts = COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:json" + ], +) diff --git a/src/snapshotcloneserver/common/define.h b/src/common/define.h similarity index 96% rename from src/snapshotcloneserver/common/define.h rename to src/common/define.h index b35e47849d..e3f90d7bd0 100644 --- a/src/snapshotcloneserver/common/define.h +++ b/src/common/define.h @@ -20,8 +20,8 @@ * Author: xuchaojie */ -#ifndef SRC_SNAPSHOTCLONESERVER_COMMON_DEFINE_H_ -#define SRC_SNAPSHOTCLONESERVER_COMMON_DEFINE_H_ +#ifndef SRC_COMMON_DEFINE_H_ +#define SRC_COMMON_DEFINE_H_ #include #include @@ -149,4 +149,4 @@ constexpr uint32_t kProgressCloneComplete = 100; } // namespace snapshotcloneserver } // namespace curve -#endif // SRC_SNAPSHOTCLONESERVER_COMMON_DEFINE_H_ +#endif // SRC_COMMON_DEFINE_H_ diff --git a/src/common/snapshotclone/BUILD b/src/common/snapshotclone/BUILD new file mode 100644 index 0000000000..f5539c00a4 --- /dev/null +++ b/src/common/snapshotclone/BUILD @@ -0,0 +1,46 @@ +# +# Copyright (c) 2020 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +COPTS = [ + "-DGFLAGS=gflags", + "-DOS_LINUX", + "-DSNAPPY", + "-DHAVE_SSE42", + "-DNDEBUG", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-msse4.2", + "-pthread", + "-Wsign-compare", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor", + "-Wno-missing-field-initializers", + "-std=c++11", +] + +cc_library( + name = "curve_snapshotclone", + srcs = glob([ + "snapshotclone_define.*", + ]), + copts = COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:json" + ], +) diff --git a/src/snapshotcloneserver/common/define.cpp b/src/common/snapshotclone/snapshotclone_define.cpp similarity index 93% rename from src/snapshotcloneserver/common/define.cpp rename to src/common/snapshotclone/snapshotclone_define.cpp index 8b47c652ca..2eb1ea1c83 100644 --- a/src/snapshotcloneserver/common/define.cpp +++ b/src/common/snapshotclone/snapshotclone_define.cpp @@ -22,7 +22,7 @@ #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" namespace curve { namespace snapshotcloneserver { @@ -40,6 +40,7 @@ const char* kCleanCloneTaskAction = "CleanCloneTask"; const char* kFlattenAction = "Flatten"; const char* kGetFileSnapshotListAction = "GetFileSnapshotList"; const char* kGetCloneTaskListAction = "GetCloneTaskList"; +const char* kGetCloneRefStatusAction = "GetCloneRefStatus"; const char* kActionStr = "Action"; const char* kVersionStr = "Version"; @@ -54,6 +55,7 @@ const char* kDestinationStr = "Destination"; const char* kLazyStr = "Lazy"; const char* kStatusStr = "Status"; const char* kTypeStr = "Type"; +const char* kInodeStr = "Inode"; const char* kCodeStr = "Code"; const char* kMessageStr = "Message"; @@ -61,6 +63,8 @@ const char* kRequestIdStr = "RequestId"; const char* kTotalCountStr = "TotalCount"; const char* kSnapshotsStr = "Snapshots"; const char* kTaskInfosStr = "TaskInfos"; +const char* kRefStatusStr = "RefStatus"; +const char* kCloneFileInfoStr = "CloneFileInfo"; std::map code2Msg = { {kErrCodeSuccess, "Exec success."}, diff --git a/src/common/snapshotclone/snapshotclone_define.h b/src/common/snapshotclone/snapshotclone_define.h new file mode 100644 index 0000000000..55925ea578 --- /dev/null +++ b/src/common/snapshotclone/snapshotclone_define.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: Mon Dec 24 2018 + * Author: xuchaojie + */ + +#ifndef SRC_COMMON_SNAPSHOTCLONE_SNAPSHOTCLONE_DEFINE_H_ +#define SRC_COMMON_SNAPSHOTCLONE_SNAPSHOTCLONE_DEFINE_H_ + +#include +#include + +namespace curve { +namespace snapshotcloneserver { + +// snapshotcloneservice字符串常量定义 +extern const char* kServiceName; +// action +extern const char* kCreateSnapshotAction; +extern const char* kDeleteSnapshotAction; +extern const char* kCancelSnapshotAction; +extern const char* kGetFileSnapshotInfoAction; +extern const char* kCloneAction; +extern const char* kRecoverAction; +extern const char* kGetCloneTasksAction; +extern const char* kCleanCloneTaskAction; +extern const char* kFlattenAction; +extern const char* kGetFileSnapshotListAction; +extern const char* kGetCloneTaskListAction; +extern const char* kGetCloneRefStatusAction; +// param +extern const char* kActionStr; +extern const char* kVersionStr; +extern const char* kUserStr; +extern const char* kFileStr; +extern const char* kNameStr; +extern const char* kUUIDStr; +extern const char* kLimitStr; +extern const char* kOffsetStr; +extern const char* kSourceStr; +extern const char* kDestinationStr; +extern const char* kLazyStr; +extern const char* kStatusStr; +extern const char* kTypeStr; +extern const char* kInodeStr; + +// json key +extern const char* kCodeStr; +extern const char* kMessageStr; +extern const char* kRequestIdStr; +extern const char* kTotalCountStr; +extern const char* kSnapshotsStr; +extern const char* kTaskInfosStr; +extern const char* kRefStatusStr; +extern const char* kCloneFileInfoStr; + +typedef std::string UUID; +using TaskIdType = UUID; + +enum class CloneTaskType { + kClone = 0, + kRecover +}; + +enum class CloneRefStatus { + kNoRef = 0, + kHasRef = 1, + kNeedCheck = 2 +}; + +// 未初始序列号 +const uint64_t kUnInitializeSeqNum = 0; +// 初始序列号 +const uint64_t kInitializeSeqNum = 1; + +// 错误码：执行成功 +const int kErrCodeSuccess = 0; +// 错误码: 内部错误 +const int kErrCodeInternalError = -1; +// 错误码：服务器初始化失败 +const int kErrCodeServerInitFail = -2; +// 错误码：服务器启动失败 +const int kErrCodeServerStartFail = -3; +// 错误码：服务已停止 +const int kErrCodeServiceIsStop = -4; +// 错误码：非法请求 +const int kErrCodeInvalidRequest = -5; +// 错误码：任务已存在 +const int kErrCodeTaskExist = -6; +// 错误码：非法的用户 +const int kErrCodeInvalidUser = -7; +// 错误码：文件不存在 +const int kErrCodeFileNotExist = -8; +// 错误码：文件状态异常 +const int kErrCodeFileStatusInvalid = -9; +// 错误码：chunk大小未按chunk分片大小对齐 +const int kErrCodeChunkSizeNotAligned = -10; +// 错误码：文件名不匹配 +const int kErrCodeFileNameNotMatch = -11; +// 错误码： 不能删除未完成的快照 +const int kErrCodeSnapshotCannotDeleteUnfinished = -12; +// 错误码: 不能对存在异常快照的文件打快照，或不能对存在错误的目标文件克隆/恢复 +const int kErrCodeSnapshotCannotCreateWhenError = -13; +// 错误码：取消的快照已完成 +const int kErrCodeCannotCancelFinished = -14; +// 错误码：不能从未完成或存在错误的快照克隆 +const int kErrCodeInvalidSnapshot = -15; +// 错误码：不能删除正在克隆的快照 +const int kErrCodeSnapshotCannotDeleteCloning = -16; +// 错误码：不能清理未完成的克隆 +const int kErrCodeCannotCleanCloneUnfinished = -17; +// 错误码：快照到达上限 +const int kErrCodeSnapshotCountReachLimit = -18; +// 错误码：文件已存在 +const int kErrCodeFileExist = -19; +// 错误码：克隆任务已满 +const int kErrCodeTaskIsFull = -20; +// 错误码：不支持 +const int kErrCodeNotSupport = -21; + +extern std::map code2Msg; + +std::string BuildErrorMessage( + int errCode, + const std::string &requestId, + const std::string &uuid = ""); + + +// clone progress +constexpr uint32_t kProgressCloneStart = 0; +constexpr uint32_t kProgressCloneError = kProgressCloneStart; +constexpr uint32_t kProgressCreateCloneFile = 1; +constexpr uint32_t kProgressCreateCloneMeta = 2; +constexpr uint32_t kProgressMetaInstalled = 5; +constexpr uint32_t kProgressRecoverChunkBegin = kProgressMetaInstalled; +constexpr uint32_t kProgressRecoverChunkEnd = 95; +constexpr uint32_t kProgressCloneComplete = 100; + + + +} // namespace snapshotcloneserver +} // namespace curve + +#endif // SRC_COMMON_SNAPSHOTCLONE_SNAPSHOTCLONE_DEFINE_H_ diff --git a/src/mds/nameserver2/BUILD b/src/mds/nameserver2/BUILD index bd5601683b..e5b7ee47ed 100644 --- a/src/mds/nameserver2/BUILD +++ b/src/mds/nameserver2/BUILD @@ -29,6 +29,7 @@ cc_library( "//src/mds/topology:topology", "//src/mds/nameserver2/allocstatistic:alloc_statistic", "//src/mds/chunkserverclient:chunkserverclient", + "//src/mds/snapshotcloneclient:snapshotcloneclient", "//src/mds/common:mds_common", "//src/mds/nameserver2/helper:helper", "//src/mds/nameserver2/idgenerator:idgenerator", diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp index ab28113163..4443a4fe58 100644 --- a/src/mds/nameserver2/curvefs.cpp +++ b/src/mds/nameserver2/curvefs.cpp @@ -23,7 +23,7 @@ #include "src/mds/nameserver2/curvefs.h" #include #include -#include +#include //NOLINT #include #include #include "src/common/string_util.h" @@ -89,7 +89,8 @@ bool CurveFS::Init(std::shared_ptr storage, std::shared_ptr fileRecordManager, std::shared_ptr allocStatistic, const struct CurveFSOption &curveFSOptions, - std::shared_ptr topology) { + std::shared_ptr topology, + std::shared_ptr snapshotCloneClient) { startTime_ = steady_clock::now(); storage_ = storage; InodeIDGenerator_ = InodeIDGenerator; @@ -101,6 +102,7 @@ bool CurveFS::Init(std::shared_ptr storage, defaultChunkSize_ = curveFSOptions.defaultChunkSize; topology_ = topology; + snapshotCloneClient_ = snapshotCloneClient; InitRootFile(); bool ret = InitRecycleBinDir(); @@ -126,6 +128,7 @@ void CurveFS::Uninit() { cleanManager_ = nullptr; allocStatistic_ = nullptr; fileRecordManager_ = nullptr; + snapshotCloneClient_ = nullptr; } void CurveFS::InitRootFile(void) { @@ -567,7 +570,24 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId, return StatusCode::kOK; } else if (fileInfo.filetype() == FileType::INODE_PAGEFILE) { StatusCode ret = CheckFileCanChange(filename, fileInfo); - if (ret != StatusCode::kOK) { + if (ret == StatusCode::kDeleteFileBeingCloned) { + bool isHasCloneRely; + StatusCode ret1 = CheckHasCloneRely(filename, + fileInfo.owner(), + &isHasCloneRely); + if (ret1 != StatusCode::kOK) { + LOG(ERROR) << "delete file, check file clone ref fail," + << "filename = " << filename + << ", ret = " << ret1; + return ret1; + } + + if (isHasCloneRely) { + LOG(WARNING) << "delete file, can not delete file, " + << "file has clone rely, filename = " << filename; + return StatusCode::kDeleteFileBeingCloned; + } + } else if (ret != StatusCode::kOK) { LOG(ERROR) << "delete file, can not delete file" << ", filename = " << filename << ", ret = " << ret; @@ -693,8 +713,8 @@ StatusCode CurveFS::CheckFileCanChange(const std::string &fileName, } if (fileInfo.filestatus() == FileStatus::kFileBeingCloned) { - LOG(ERROR) << "CheckFileCanChange, file is being Cloned, " - << "cannot delete or rename, fileName = " << fileName; + LOG(WARNING) << "CheckFileCanChange, file is being Cloned, " + << "need check first, fileName = " << fileName; return StatusCode::kDeleteFileBeingCloned; } @@ -1813,6 +1833,51 @@ StatusCode CurveFS::ListClient(bool listAllClient, return StatusCode::kOK; } +StatusCode CurveFS::CheckHasCloneRely(const std::string & filename, + const std::string &owner, + bool *isHasCloneRely) { + CloneRefStatus refStatus; + std::vector fileCheckList; + StatusCode ret = snapshotCloneClient_->GetCloneRefStatus(filename, + owner, &refStatus, &fileCheckList); + if (ret != StatusCode::kOK) { + LOG(ERROR) << "delete file, check file clone ref fail," + << "filename = " << filename + << ", ret = " << ret; + return ret; + } + bool hasCloneRef = false; + if (refStatus == CloneRefStatus::kHasRef) { + hasCloneRef = true; + } else if (refStatus == CloneRefStatus::kNoRef) { + hasCloneRef = false; + } else { + int recordNum = fileCheckList.size(); + for (int i = 0; i < recordNum; i++) { + FileInfo destFileInfo; + StatusCode ret2 = GetFileInfo(fileCheckList[i].filename, + &destFileInfo); + if (ret2 == StatusCode::kFileNotExists) { + continue; + } else if (ret2 == StatusCode::kOK) { + if (destFileInfo.id() == fileCheckList[i].inodeid) { + hasCloneRef = true; + break; + } + } else { + LOG(ERROR) << "CheckHasCloneRely, check clonefile exist fail" + << ", filename = " << filename + << ", clonefile = " << fileCheckList[i].filename + << ", ret = " << ret2; + return ret2; + } + } + } + + *isHasCloneRely = hasCloneRef; + return StatusCode::kOK; +} + StatusCode CurveFS::FindFileMountPoint( const std::string& fileName, ClientInfo* clientInfo) { diff --git a/src/mds/nameserver2/curvefs.h b/src/mds/nameserver2/curvefs.h index 7e33c9b811..b005f080b0 100644 --- a/src/mds/nameserver2/curvefs.h +++ b/src/mds/nameserver2/curvefs.h @@ -28,7 +28,7 @@ #include #include #include //NOLINT -#include +#include //NOLINT #include #include "proto/nameserver2.pb.h" #include "src/mds/nameserver2/namespace_storage.h" @@ -40,7 +40,9 @@ #include "src/mds/nameserver2/idgenerator/inode_id_generator.h" #include "src/common/authenticator.h" #include "src/mds/nameserver2/allocstatistic/alloc_statistic.h" +#include "src/mds/snapshotcloneclient/snapshotclone_client.h" using curve::common::Authenticator; +using curve::mds::snapshotcloneclient::SnapshotCloneClient; namespace curve { namespace mds { @@ -93,7 +95,8 @@ class CurveFS { std::shared_ptr fileRecordManager, std::shared_ptr allocStatistic, const struct CurveFSOption &curveFSOptions, - std::shared_ptr topology); + std::shared_ptr topology, + std::shared_ptr snapshotCloneClient); /** * @brief Run session manager @@ -599,6 +602,17 @@ class CurveFS { const FileInfo& fileInfo, uint64_t* fileSize); + /** + * @brief check file has rely dest file + * @param: fileName + * @param: owner + * @param[out]: isCloneHasRely: is clone has rely + * @return StatusCode::kOK if succeeded + */ + StatusCode CheckHasCloneRely(const std::string & filename, + const std::string &owner, + bool *isHasCloneRely); + private: FileInfo rootFileInfo_; std::shared_ptr storage_; @@ -608,6 +622,7 @@ class CurveFS { std::shared_ptr cleanManager_; std::shared_ptr allocStatistic_; std::shared_ptr topology_; + std::shared_ptr snapshotCloneClient_; struct RootAuthOption rootAuthOptions_; uint64_t defaultChunkSize_; diff --git a/src/mds/server/mds.cpp b/src/mds/server/mds.cpp index 1eaff07d2d..6a3bc05e96 100644 --- a/src/mds/server/mds.cpp +++ b/src/mds/server/mds.cpp @@ -49,6 +49,7 @@ void MDS::InitMdsOptions(std::shared_ptr conf) { InitTopologyOption(&options_.topologyOption); InitCopysetOption(&options_.copysetOption); InitChunkServerClientOption(&options_.chunkServerClientOption); + InitSnapshotCloneClientOption(&options_.snapshotCloneClientOption); conf_->GetValueFatalIfFail( "mds.segment.alloc.retryInterMs", &options_.retryInterTimes); @@ -388,6 +389,20 @@ void MDS::InitNameServerStorage(int mdsCacheCount) { LOG(INFO) << "init NameServerStorage success."; } +void MDS::InitSnapshotCloneClientOption(SnapshotCloneClientOption *option) { + if (!conf_->GetValue("mds.snapshotcloneclient.addr", + &option->snapshotCloneAddr)) { + option->snapshotCloneAddr = ""; + } +} + +void MDS::InitSnapshotCloneClient() { + snapshotCloneClient_ = std::make_shared(); + SnapshotCloneClientOption snapshotCloneClientOption; + InitSnapshotCloneClientOption(&snapshotCloneClientOption); + snapshotCloneClient_->Init(snapshotCloneClientOption); +} + void MDS::InitCurveFS(const CurveFSOption& curveFSOptions) { // init InodeIDGenerator auto inodeIdGenerator = std::make_shared(etcdClient_); @@ -406,11 +421,16 @@ void MDS::InitCurveFS(const CurveFSOption& curveFSOptions) { // init FileRecordManager auto fileRecordManager = std::make_shared(); + + // init snapshotCloneClient + InitSnapshotCloneClient(); + LOG_IF(FATAL, !kCurveFS.Init(nameServerStorage_, inodeIdGenerator, chunkSegmentAllocate, cleanManager_, fileRecordManager, segmentAllocStatistic_, - curveFSOptions, topology_)) + curveFSOptions, topology_, + snapshotCloneClient_)) << "init FileRecordManager fail"; LOG(INFO) << "init FileRecordManager success."; diff --git a/src/mds/server/mds.h b/src/mds/server/mds.h index 01dda447d2..c4b1ed9289 100644 --- a/src/mds/server/mds.h +++ b/src/mds/server/mds.h @@ -75,6 +75,7 @@ using ::curve::mds::schedule::ScheduleOption; using ::curve::mds::schedule::ScheduleMetrics; using ::curve::mds::schedule::ScheduleServiceImpl; using ::curve::mds::chunkserverclient::ChunkServerClientOption; +using ::curve::mds::snapshotcloneclient::SnapshotCloneClientOption; using ::curve::election::LeaderElectionOptions; using ::curve::election::LeaderElection; using ::curve::common::Configuration; @@ -102,6 +103,7 @@ struct MDSOptions { TopologyOption topologyOption; CopysetOption copysetOption; ChunkServerClientOption chunkServerClientOption; + SnapshotCloneClientOption snapshotCloneClientOption; }; class MDS { @@ -117,7 +119,7 @@ class MDS { /** * @brief start MDS DummyServer for liveness probe for all mds - * and get metrics like version and configuration + * and get metrics like version and configuration */ void StartDummy(); @@ -163,6 +165,8 @@ class MDS { void InitChunkServerClientOption(ChunkServerClientOption *option); + void InitSnapshotCloneClientOption(SnapshotCloneClientOption *option); + void InitEtcdClient(const EtcdConf& etcdConf, int etcdTimeout, int retryTimes); @@ -196,6 +200,8 @@ class MDS { void InitHeartbeatManager(); + void InitSnapshotCloneClient(); + private: // mds configuration items std::shared_ptr conf_; @@ -221,6 +227,7 @@ class MDS { std::shared_ptr heartbeatManager_; char* etcdEndpoints_; FileLockManager* fileLockManager_; + std::shared_ptr snapshotCloneClient_; }; } // namespace mds diff --git a/src/mds/snapshotcloneclient/BUILD b/src/mds/snapshotcloneclient/BUILD new file mode 100644 index 0000000000..023da01367 --- /dev/null +++ b/src/mds/snapshotcloneclient/BUILD @@ -0,0 +1,38 @@ +# +# Copyright (c) 2020 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load( + "//:copts.bzl", + "GCC_FLAGS", + "GCC_TEST_FLAGS", +) + +cc_library( + name = "snapshotcloneclient", + srcs = glob([ + "*.cpp", + "*.h", + ]), + copts = GCC_FLAGS, + visibility = ["//visibility:public"], + deps = [ + "//external:brpc", + "//external:glog", + "//external:json", + "//src/common/snapshotclone:curve_snapshotclone", + "//proto:nameserver2_cc_proto", + ], +) diff --git a/src/mds/snapshotcloneclient/snapshotclone_client.cpp b/src/mds/snapshotcloneclient/snapshotclone_client.cpp new file mode 100644 index 0000000000..3f76d111a7 --- /dev/null +++ b/src/mds/snapshotcloneclient/snapshotclone_client.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2020/11/27 + * Author: hzchenwei7 + */ + +#include "src/mds/snapshotcloneclient/snapshotclone_client.h" +#include +#include + +using curve::snapshotcloneserver::kServiceName; +using curve::snapshotcloneserver::kActionStr; +using curve::snapshotcloneserver::kGetCloneRefStatusAction; +using curve::snapshotcloneserver::kVersionStr; +using curve::snapshotcloneserver::kUserStr; +using curve::snapshotcloneserver::kSourceStr; +using curve::snapshotcloneserver::kCodeStr; +using curve::snapshotcloneserver::kRefStatusStr; +using curve::snapshotcloneserver::kTotalCountStr; +using curve::snapshotcloneserver::kCloneFileInfoStr; +using curve::snapshotcloneserver::kFileStr; +using curve::snapshotcloneserver::kInodeStr; + + +namespace curve { +namespace mds { +namespace snapshotcloneclient { + +StatusCode SnapshotCloneClient::GetCloneRefStatus(std::string filename, + std::string user, + CloneRefStatus *status, + std::vector *fileCheckList) { + if (!inited_) { + LOG(WARNING) << "GetCloneRefStatus, snapshot clone server not inited" + << ", filename = " << filename + << ", user = " << user; + return StatusCode::kSnapshotCloneServerNotInit; + } + + brpc::Channel channel; + brpc::ChannelOptions option; + option.protocol = "http"; + + std::string url = addr_ + + "/" + kServiceName + "?" + + kActionStr+ "=" + kGetCloneRefStatusAction + "&" + + kVersionStr + "=1&" + + kUserStr + "=" + user + "&" + + kSourceStr + "=" + filename; + + if (channel.Init(url.c_str(), "", &option) != 0) { + LOG(ERROR) << "GetCloneRefStatus, Fail to init channel, url is " << url + << ", filename = " << filename + << ", user = " << user; + return StatusCode::kSnapshotCloneConnectFail; + } + + brpc::Controller cntl; + cntl.http_request().uri() = url.c_str(); + + channel.CallMethod(NULL, &cntl, NULL, NULL, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "GetCloneRefStatus, CallMethod faile, errMsg :" + << cntl.ErrorText() + << ", filename = " << filename + << ", user = " << user; + return StatusCode::KInternalError; + } + + std::stringstream ss; + ss << cntl.response_attachment(); + std::string data = ss.str(); + Json::Reader jsonReader; + Json::Value jsonObj; + if (!jsonReader.parse(data, jsonObj)) { + LOG(ERROR) << "GetCloneRefStatus, parse json fail, data = " << data + << ", filename = " << filename + << ", user = " << user; + return StatusCode::KInternalError; + } + + LOG(INFO) << "GetCloneRefStatus, " << data; + + std::string requestCode = jsonObj[kCodeStr].asCString(); + if (requestCode != "0") { + LOG(ERROR) << "GetCloneRefStatus, Code is not 0, data = " << data + << ", filename = " << filename + << ", user = " << user; + return StatusCode::KInternalError; + } + + CloneRefStatus tempStatus = + static_cast(jsonObj[kRefStatusStr].asInt()); + *status = tempStatus; + if (tempStatus == CloneRefStatus::kNoRef + || tempStatus == CloneRefStatus::kHasRef) { + return StatusCode::kOK; + } + + if (tempStatus != CloneRefStatus::kNeedCheck) { + LOG(ERROR) << "GetCloneRefStatus, invalid status, data = " << data + << ", filename = " << filename + << ", user = " << user; + return StatusCode::KInternalError; + } + + int totalCount = jsonObj[kTotalCountStr].asInt(); + int listSize = jsonObj[kCloneFileInfoStr].size(); + for (int i = 0; i < listSize; i++) { + DestFileInfo file; + file.filename = jsonObj[kCloneFileInfoStr][i][kFileStr].asCString(); + file.inodeid = jsonObj[kCloneFileInfoStr][i][kInodeStr].asUInt64(); + fileCheckList->push_back(file); + } + return StatusCode::kOK; +} + +void SnapshotCloneClient::Init(const SnapshotCloneClientOption &option) { + if (!option.snapshotCloneAddr.empty()) { + addr_ = option.snapshotCloneAddr; + inited_ = true; + } +} + +bool SnapshotCloneClient::GetInitStatus() { + return inited_; +} +} // namespace snapshotcloneclient +} // namespace mds +} // namespace curve + diff --git a/src/mds/snapshotcloneclient/snapshotclone_client.h b/src/mds/snapshotcloneclient/snapshotclone_client.h new file mode 100644 index 0000000000..525f403554 --- /dev/null +++ b/src/mds/snapshotcloneclient/snapshotclone_client.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2020/11/26 + * Author: hzchenwei7 + */ + +#ifndef SRC_MDS_SNAPSHOTCLONECLIENT_SNAPSHOTCLONE_CLIENT_H_ +#define SRC_MDS_SNAPSHOTCLONECLIENT_SNAPSHOTCLONE_CLIENT_H_ + +#include +#include +#include "src/common/snapshotclone/snapshotclone_define.h" +#include "proto/nameserver2.pb.h" // for retcode StatusCode + +using curve::mds::StatusCode; +using curve::snapshotcloneserver::CloneRefStatus; + +namespace curve { +namespace mds { +namespace snapshotcloneclient { + +struct SnapshotCloneClientOption { + std::string snapshotCloneAddr; +}; + +struct DestFileInfo { + std::string filename; + uint64_t inodeid; +}; + +class SnapshotCloneClient { + public: + SnapshotCloneClient() + : addr_(""), inited_(false) {} + + virtual ~SnapshotCloneClient() {} + + virtual void Init(const SnapshotCloneClientOption &option); + + /** + * @brief get the clone ref status of a file. As a clone src file, + * find if has clone ref. + * + * @param filename the name of clone src file + * @param user the user of clone src file + * @param status return 0 means no ref, return 1 means has ref, + * return 2 means not sure, need check fileCheckList + * @param fileCheckList the need check dest file info list + * + * @return error code + */ + virtual StatusCode GetCloneRefStatus(std::string filename, std::string user, + CloneRefStatus *status, + std::vector *fileCheckList); + + // test only + virtual bool GetInitStatus(); + + private: + std::string addr_; + bool inited_; +}; + +} // namespace snapshotcloneclient +} // namespace mds +} // namespace curve + + +#endif // SRC_MDS_SNAPSHOTCLONECLIENT_SNAPSHOTCLONE_CLIENT_H_ diff --git a/src/snapshotcloneserver/BUILD b/src/snapshotcloneserver/BUILD index 227fa2d57f..39c177c378 100644 --- a/src/snapshotcloneserver/BUILD +++ b/src/snapshotcloneserver/BUILD @@ -70,6 +70,7 @@ cc_library( "//proto:snapshotcloneserver_cc_proto", "//src/common:curve_common", "//src/common:curve_s3_adapter", + "//src/common/snapshotclone:curve_snapshotclone", "//proto:nameserver2_cc_proto", "//proto:chunkserver-cc-protos", "//src/client:curve_client", @@ -111,6 +112,7 @@ cc_binary( "//proto:snapshotcloneserver_cc_proto", "//src/common:curve_common", "//src/common:curve_s3_adapter", + "//src/common/snapshotclone:curve_snapshotclone", "//proto:nameserver2_cc_proto", "//proto:chunkserver-cc-protos", "//src/client:curve_client", diff --git a/src/snapshotcloneserver/clone/clone_closure.h b/src/snapshotcloneserver/clone/clone_closure.h index 92f11a538e..fd812c9a1b 100644 --- a/src/snapshotcloneserver/clone/clone_closure.h +++ b/src/snapshotcloneserver/clone/clone_closure.h @@ -29,7 +29,7 @@ #include #include "proto/snapshotcloneserver.pb.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "json/json.h" #include "src/common/concurrent/name_lock.h" diff --git a/src/snapshotcloneserver/clone/clone_core.cpp b/src/snapshotcloneserver/clone/clone_core.cpp index 4355f5e54d..ef799b9fda 100644 --- a/src/snapshotcloneserver/clone/clone_core.cpp +++ b/src/snapshotcloneserver/clone/clone_core.cpp @@ -1571,6 +1571,93 @@ int CloneCoreImpl::HandleRemoveCloneOrRecoverTask( << ", taskId = " << taskId; return kErrCodeInternalError; } + + if (IsSnapshot(task)) { + snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); + } else { + std::string source = task->GetCloneInfo().GetSrc(); + cloneRef_->DecrementRef(source); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) { + int ret = client_->SetCloneFileStatus(source, + FileStatus::Created, mdsRootUser_); + if (ret < 0) { + LOG(ERROR) << "Task Fail cause by SetCloneFileStatus fail" + << ", ret = " << ret + << ", TaskInfo : " << *task; + return kErrCodeInternalError; + } + } + } + + return kErrCodeSuccess; +} + +int CloneCoreImpl::CheckFileExists(const std::string &filename, + uint64_t inodeId) { + FInfo destFInfo; + int ret = client_->GetFileInfo(filename, mdsRootUser_, &destFInfo); + if (ret == LIBCURVE_ERROR::OK) { + if (destFInfo.id == inodeId) { + return kErrCodeFileExist; + } else { + return kErrCodeFileNotExist; + } + } + + if (ret == -LIBCURVE_ERROR::NOTEXIST) { + return kErrCodeFileNotExist; + } + + return kErrCodeInternalError; +} + +// 加减引用计数的时候，接口里面会对引用计数map加锁； +// 加引用计数、处理引用计数减到0的时候，需要额外对修改的那条记录加锁。 +int CloneCoreImpl::HandleDeleteCloneInfo(const CloneInfo &cloneInfo) { + // 先减引用计数，如果是从镜像克隆且引用计数减到0，需要修改源镜像的状态为created + std::string source = cloneInfo.GetSrc(); + if (cloneInfo.GetFileType() == CloneFileType::kSnapshot) { + snapshotRef_->DecrementSnapshotRef(source); + } else { + cloneRef_->DecrementRef(source); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) { + int ret = client_->SetCloneFileStatus(source, + FileStatus::Created, mdsRootUser_); + if (ret == -LIBCURVE_ERROR::NOTEXIST) { + LOG(WARNING) << "SetCloneFileStatus, file not exist, filename: " + << source; + } else if (ret != LIBCURVE_ERROR::OK) { + cloneRef_->IncrementRef(source); + LOG(ERROR) << "SetCloneFileStatus fail" + << ", ret = " << ret + << ", cloneInfo : " << cloneInfo; + return kErrCodeInternalError; + } + } + } + + // 删除这条记录，如果删除失败，把前面已经减掉的引用计数加回去 + int ret = metaStore_->DeleteCloneInfo(cloneInfo.GetTaskId()); + if (ret != 0) { + if (cloneInfo.GetFileType() == CloneFileType::kSnapshot) { + NameLockGuard lockSnapGuard(snapshotRef_->GetSnapshotLock(), + source); + snapshotRef_->IncrementSnapshotRef(source); + } else { + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + cloneRef_->IncrementRef(source); + } + LOG(ERROR) << "DeleteCloneInfo failed" + << ", ret = " << ret + << ", CloneInfo = " << cloneInfo; + return kErrCodeInternalError; + } + + LOG(INFO) << "HandleDeleteCloneInfo success" + << ", cloneInfo = " << cloneInfo; + return kErrCodeSuccess; } diff --git a/src/snapshotcloneserver/clone/clone_core.h b/src/snapshotcloneserver/clone/clone_core.h index 1551a70e14..67332b5b89 100644 --- a/src/snapshotcloneserver/clone/clone_core.h +++ b/src/snapshotcloneserver/clone/clone_core.h @@ -30,7 +30,7 @@ #include #include "src/snapshotcloneserver/common/curvefs_client.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/snapshotclone_meta_store.h" #include "src/snapshotcloneserver/snapshot/snapshot_data_store.h" #include "src/snapshotcloneserver/common/snapshot_reference.h" @@ -173,6 +173,25 @@ class CloneCore { */ virtual int HandleRemoveCloneOrRecoverTask( std::shared_ptr task) = 0; + + /** + * @brief 检查文件是否存在 + * + * @param filename 文件名 + * + * @return 错误码 + */ + virtual int CheckFileExists(const std::string &filename, + uint64_t inodeId) = 0; + + /** + * @brief 删除cloneInfo + * + * @param cloneInfo 待删除的cloneInfo + * + * @return 错误码 + */ + virtual int HandleDeleteCloneInfo(const CloneInfo &cloneInfo) = 0; }; /** @@ -263,6 +282,10 @@ class CloneCoreImpl : public CloneCore { int HandleRemoveCloneOrRecoverTask( std::shared_ptr task) override; + int CheckFileExists(const std::string &filename, + uint64_t inodeId) override; + int HandleDeleteCloneInfo(const CloneInfo &cloneInfo) override; + private: /** * @brief 从快照构建克隆/恢复的文件信息 diff --git a/src/snapshotcloneserver/clone/clone_reference.h b/src/snapshotcloneserver/clone/clone_reference.h index 9b03952aff..8b97cea1d2 100644 --- a/src/snapshotcloneserver/clone/clone_reference.h +++ b/src/snapshotcloneserver/clone/clone_reference.h @@ -27,7 +27,7 @@ #include #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/rw_lock.h" #include "src/common/concurrent/name_lock.h" diff --git a/src/snapshotcloneserver/clone/clone_service_manager.cpp b/src/snapshotcloneserver/clone/clone_service_manager.cpp index e5cd1750fe..3eb4620265 100644 --- a/src/snapshotcloneserver/clone/clone_service_manager.cpp +++ b/src/snapshotcloneserver/clone/clone_service_manager.cpp @@ -29,7 +29,7 @@ #include #include "src/snapshotcloneserver/common/snapshotclone_metric.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/common/string_util.h" namespace curve { @@ -42,16 +42,22 @@ int CloneServiceManager::Init(const SnapshotCloneServerOptions &option) { std::make_shared(option.stage2PoolThreadNum); std::shared_ptr commonPool = std::make_shared(option.commonPoolThreadNum); + cloneServiceManagerBackend_->Init( + option.backEndReferenceRecordScanIntervalMs, + option.backEndReferenceFuncScanIntervalMs); return cloneTaskMgr_->Init(stage1Pool, stage2Pool, commonPool, option); } int CloneServiceManager::Start() { + cloneServiceManagerBackend_->Start(); return cloneTaskMgr_->Start(); } void CloneServiceManager::Stop() { cloneTaskMgr_->Stop(); + cloneServiceManagerBackend_->Stop(); } + int CloneServiceManager::CloneFile(const UUID &source, const std::string &user, const std::string &destination, @@ -300,6 +306,48 @@ int CloneServiceManager::GetCloneTaskInfoByFilter( return GetCloneTaskInfoInner(cloneInfos, filter, info); } +int CloneServiceManager::GetCloneRefStatus(const std::string &src, + CloneRefStatus *refStatus, + std::vector *needCheckFiles) { + std::vector cloneInfos; + int ret = cloneCore_->GetCloneInfoList(&cloneInfos); + if (ret < 0) { + *refStatus = CloneRefStatus::kNoRef; + return kErrCodeSuccess; + } + + *refStatus = CloneRefStatus::kNoRef; + for (auto &cloneInfo : cloneInfos) { + if (cloneInfo.GetSrc() == src) { + switch (cloneInfo.GetStatus()) { + case CloneStatus::done : + case CloneStatus::error: { + break; + } + case CloneStatus::cleaning: + case CloneStatus::errorCleaning: + case CloneStatus::retrying: + case CloneStatus::cloning: + case CloneStatus::recovering: { + *refStatus = CloneRefStatus::kHasRef; + needCheckFiles->clear(); + return kErrCodeSuccess; + } + case CloneStatus::metaInstalled: { + *refStatus = CloneRefStatus::kNeedCheck; + needCheckFiles->emplace_back(cloneInfo); + break; + } + default: + LOG(ERROR) << "can not reach here!, status = " + << static_cast(cloneInfo.GetStatus()); + return kErrCodeInternalError; + } + } + } + return kErrCodeSuccess; +} + int CloneServiceManager::GetCloneTaskInfoInner( std::vector cloneInfos, CloneFilterCondition filter, @@ -643,6 +691,101 @@ int CloneServiceManager::RecoverCloneTask() { return kErrCodeSuccess; } +// 当clone处于matainstall状态，且克隆卷已经删除的情况下，原卷的引用计数没有减。 +// 这个后台线程处理函数周期性的检查这个场景，如果发现有clone处于metaintalled状态 +// 且克隆卷已经删除，就去删除这条无效的clone信息，并减去原卷的引用计数。 +// 如果原卷是镜像且引用计数减为0，还需要去mds把原卷的状态改为created。 +void CloneServiceManagerBackendImpl::Func() { + LOG(INFO) << "CloneServiceManager BackEndReferenceScanFunc start"; + while (!isStop_.load()) { + std::vector cloneInfos; + int ret = cloneCore_->GetCloneInfoList(&cloneInfos); + if (ret < 0) { + LOG(WARNING) << "GetCloneInfoList fail" << ", ret = " << ret; + } + + int deleteCount = 0; + for (auto &it : cloneInfos) { + if (it.GetStatus() == CloneStatus::metaInstalled + && it.GetIsLazy() == true) { + // 检查destination在不在 + if (it.GetTaskType() == CloneTaskType::kClone) { + ret = cloneCore_->CheckFileExists(it.GetDest(), + it.GetDestId()); + } else { + // rename时，inodeid恢复成 + ret = cloneCore_->CheckFileExists(it.GetDest(), + it.GetOriginId()); + } + + if (ret == kErrCodeFileNotExist) { + // 如果克隆卷是metaInstalled状态，且destination文件不存在， + // 删除这条cloneInfo，并减引用计数 + TaskIdType taskId = it.GetTaskId(); + CloneInfo cloneInfo; + ret = cloneCore_->GetCloneInfo(taskId, &cloneInfo); + if (ret != kErrCodeSuccess) { + // cloneInfo已经不存在了 + continue; + } + + // 再次检查cloneInfo是否是metaInstalled状态 + if (cloneInfo.GetStatus() != CloneStatus::metaInstalled) { + continue; + } + ret = cloneCore_->HandleDeleteCloneInfo(cloneInfo); + if (ret != kErrCodeSuccess) { + LOG(WARNING) << "HandleDeleteCloneInfo fail, ret = " + << ret << ", cloneInfo = " << cloneInfo; + } else { + deleteCount++; + } + } + + recordWaitInterval_.WaitForNextExcution(); + } + } + + LOG(INFO) << "backend scan list, size = " << cloneInfos.size() + << ", delete clone record count = " << deleteCount; + + // 控制每轮扫描间隔 + roundWaitInterval_.WaitForNextExcution(); + } + LOG(INFO) << "CloneServiceManager BackEndReferenceScanFunc exit"; +} + +void CloneServiceManagerBackendImpl::Init(uint32_t recordIntevalMs, + uint32_t roundIntevalMs) { + recordWaitInterval_.Init(recordIntevalMs); + roundWaitInterval_.Init(roundIntevalMs); + + LOG(INFO) << "Init recordIntevalMs = " << recordIntevalMs + << ", roundIntevalMs = " << roundIntevalMs; + return; +} + +void CloneServiceManagerBackendImpl::Start() { + if (isStop_.load()) { + isStop_.store(false); + backEndReferenceScanThread_ = + std::thread(&CloneServiceManagerBackendImpl::Func, this); + } + return; +} + +void CloneServiceManagerBackendImpl::Stop() { + recordWaitInterval_.StopWait(); + roundWaitInterval_.StopWait(); + if (!isStop_.exchange(true)) { + if (backEndReferenceScanThread_.joinable()) { + backEndReferenceScanThread_.join(); + } + } + + return; +} + } // namespace snapshotcloneserver } // namespace curve diff --git a/src/snapshotcloneserver/clone/clone_service_manager.h b/src/snapshotcloneserver/clone/clone_service_manager.h index 519300be3d..754e76fa30 100644 --- a/src/snapshotcloneserver/clone/clone_service_manager.h +++ b/src/snapshotcloneserver/clone/clone_service_manager.h @@ -26,10 +26,11 @@ #include #include +#include "src/common/wait_interval.h" #include "src/snapshotcloneserver/clone/clone_core.h" #include "src/snapshotcloneserver/clone/clone_task.h" #include "src/snapshotcloneserver/clone/clone_task_manager.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "src/snapshotcloneserver/clone/clone_closure.h" @@ -151,14 +152,61 @@ class CloneFilterCondition { const std::string *status_; const std::string *type_; }; +class CloneServiceManagerBackend { + public: + CloneServiceManagerBackend() {} + virtual ~CloneServiceManagerBackend() {} + + /** + * @brief 后台扫描线程执行函数，扫描克隆卷是否存在 + * + */ + virtual void Func() = 0; + + virtual void Init(uint32_t recordIntevalMs, uint32_t roundIntevalMs) = 0; + + virtual void Start() = 0; + + virtual void Stop() = 0; +}; + +class CloneServiceManagerBackendImpl : public CloneServiceManagerBackend { + public: + explicit CloneServiceManagerBackendImpl( + std::shared_ptr cloneCore) + : cloneCore_(cloneCore), + isStop_(true) { + } + + ~CloneServiceManagerBackendImpl() { + } + + void Func() override; + void Init(uint32_t recordIntevalMs, uint32_t roundIntevalMs) override; + void Start() override; + void Stop() override; + + private: + std::shared_ptr cloneCore_; + // 后台扫描线程，扫描clone卷是否存在 + std::thread backEndReferenceScanThread_; + // 当前后台扫描是否停止，用于支持start，stop功能 + std::atomic_bool isStop_; + // 后台扫描线程记录使用定时器 + common::WaitInterval recordWaitInterval_; + // 后台扫描线程每轮使用定时器 + common::WaitInterval roundWaitInterval_; +}; class CloneServiceManager { public: CloneServiceManager( std::shared_ptr cloneTaskMgr, - std::shared_ptr cloneCore) + std::shared_ptr cloneCore, + std::shared_ptr cloneServiceManagerBackend) : cloneTaskMgr_(cloneTaskMgr), - cloneCore_(cloneCore) { + cloneCore_(cloneCore), + cloneServiceManagerBackend_(cloneServiceManagerBackend) { destFileLock_ = std::make_shared(); } virtual ~CloneServiceManager() {} @@ -283,6 +331,19 @@ class CloneServiceManager { virtual int GetCloneTaskInfoByFilter(const CloneFilterCondition &filter, std::vector *info); + /** + * @brief 查询src是否有依赖 + * + * @param src 指定的文件名 + * @param refStatus 0表示没有依赖，1表示有依赖，2表示需要进一步确认 + * @param needCheckFiles 需要进一步确认的文件列表 + * + * @return 错误码 + */ + virtual int GetCloneRefStatus(const std::string &src, + CloneRefStatus *refStatus, + std::vector *needCheckFiles); + /** * @brief 清除失败的clone/Recover任务、状态、文件 * @@ -386,8 +447,11 @@ class CloneServiceManager { std::shared_ptr destFileLock_; std::shared_ptr cloneTaskMgr_; std::shared_ptr cloneCore_; + std::shared_ptr cloneServiceManagerBackend_; }; + + } // namespace snapshotcloneserver } // namespace curve diff --git a/src/snapshotcloneserver/clone/clone_task.h b/src/snapshotcloneserver/clone/clone_task.h index c65b05b392..2f7b5bf118 100644 --- a/src/snapshotcloneserver/clone/clone_task.h +++ b/src/snapshotcloneserver/clone/clone_task.h @@ -27,7 +27,7 @@ #include #include "src/snapshotcloneserver/clone/clone_core.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/task.h" #include "src/snapshotcloneserver/common/task_info.h" #include "src/snapshotcloneserver/common/snapshotclone_metric.h" diff --git a/src/snapshotcloneserver/clone/clone_task_manager.cpp b/src/snapshotcloneserver/clone/clone_task_manager.cpp index 19c25446d1..be14fc5db6 100644 --- a/src/snapshotcloneserver/clone/clone_task_manager.cpp +++ b/src/snapshotcloneserver/clone/clone_task_manager.cpp @@ -21,7 +21,7 @@ */ #include "src/snapshotcloneserver/clone/clone_task_manager.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" namespace curve { diff --git a/src/snapshotcloneserver/clone/clone_task_manager.h b/src/snapshotcloneserver/clone/clone_task_manager.h index a5e4e7025c..d9607ccedc 100644 --- a/src/snapshotcloneserver/clone/clone_task_manager.h +++ b/src/snapshotcloneserver/clone/clone_task_manager.h @@ -33,7 +33,7 @@ #include "src/snapshotcloneserver/clone/clone_task.h" #include "src/snapshotcloneserver/common/thread_pool.h" #include "src/common/concurrent/rw_lock.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "src/snapshotcloneserver/common/snapshotclone_metric.h" #include "src/snapshotcloneserver/clone/clone_core.h" @@ -149,11 +149,11 @@ class CloneTaskManager { // 存放stage1Poo2_池的当前任务，key为destination std::map > stage2TaskMap_; - mutable Mutex stage2TasksLock_;; + mutable Mutex stage2TasksLock_; // 存放commonPool_池的当前任务 std::map > commonTaskMap_; - mutable Mutex commonTasksLock_;; + mutable Mutex commonTasksLock_; // 用于Lazy克隆元数据部分的线程池 std::shared_ptr stage1Pool_; diff --git a/src/snapshotcloneserver/common/config.h b/src/snapshotcloneserver/common/config.h index 2b16420d06..246e7bee18 100644 --- a/src/snapshotcloneserver/common/config.h +++ b/src/snapshotcloneserver/common/config.h @@ -86,6 +86,10 @@ struct SnapshotCloneServerOptions { uint32_t createCloneChunkConcurrency; // RecoverChunk同时进行的异步请求数量 uint32_t recoverChunkConcurrency; + // 引用计数后台扫描每条记录间隔 + uint32_t backEndReferenceRecordScanIntervalMs; + // 引用计数后台扫描每轮间隔 + uint32_t backEndReferenceFuncScanIntervalMs; }; } // namespace snapshotcloneserver diff --git a/src/snapshotcloneserver/common/curvefs_client.h b/src/snapshotcloneserver/common/curvefs_client.h index 5538a48e1b..b5686f9a59 100644 --- a/src/snapshotcloneserver/common/curvefs_client.h +++ b/src/snapshotcloneserver/common/curvefs_client.h @@ -35,7 +35,7 @@ #include "src/client/client_common.h" #include "src/client/libcurve_snapshot.h" #include "src/client/libcurve_file.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "src/common/timeutility.h" diff --git a/src/snapshotcloneserver/common/snapshot_reference.h b/src/snapshotcloneserver/common/snapshot_reference.h index a4451f6b0c..d828ab705c 100644 --- a/src/snapshotcloneserver/common/snapshot_reference.h +++ b/src/snapshotcloneserver/common/snapshot_reference.h @@ -26,7 +26,7 @@ #include #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/rw_lock.h" #include "src/common/concurrent/name_lock.h" diff --git a/src/snapshotcloneserver/common/snapshotclone_info.h b/src/snapshotcloneserver/common/snapshotclone_info.h index bc461ef58a..62a14054c4 100644 --- a/src/snapshotcloneserver/common/snapshotclone_info.h +++ b/src/snapshotcloneserver/common/snapshotclone_info.h @@ -28,7 +28,7 @@ #include #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" namespace curve { namespace snapshotcloneserver { diff --git a/src/snapshotcloneserver/common/snapshotclone_meta_store.h b/src/snapshotcloneserver/common/snapshotclone_meta_store.h index 48cf91de3f..4facef3146 100644 --- a/src/snapshotcloneserver/common/snapshotclone_meta_store.h +++ b/src/snapshotcloneserver/common/snapshotclone_meta_store.h @@ -29,7 +29,7 @@ #include #include //NOLINT -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "src/common/concurrent/concurrent.h" #include "src/snapshotcloneserver/common/snapshotclone_info.h" diff --git a/src/snapshotcloneserver/common/snapshotclone_meta_store_etcd.cpp b/src/snapshotcloneserver/common/snapshotclone_meta_store_etcd.cpp index c3a9cda2ba..c0c4a4f9bd 100644 --- a/src/snapshotcloneserver/common/snapshotclone_meta_store_etcd.cpp +++ b/src/snapshotcloneserver/common/snapshotclone_meta_store_etcd.cpp @@ -200,7 +200,17 @@ int SnapshotCloneMetaStoreEtcd::UpdateCloneInfo(const CloneInfo &info) { return -1; } WriteLockGuard guard(cloneInfos_lock_); - int errCode = client_->Put(key, value); + // if old record not exist, return failed + std::string oldValue; + int errCode = client_->Get(key, &oldValue); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "Get old cloneInfo from etcd err" + << ", errcode = " << errCode + << ", cloneInfo : " << info; + return -1; + } + + errCode = client_->Put(key, value); if (errCode != EtcdErrCode::EtcdOK) { LOG(ERROR) << "Put cloneInfo into etcd err" << ", errcode = " << errCode @@ -211,7 +221,8 @@ int SnapshotCloneMetaStoreEtcd::UpdateCloneInfo(const CloneInfo &info) { if (search != cloneInfos_.end()) { search->second = info; } else { - cloneInfos_.emplace(info.GetTaskId(), info); + LOG(ERROR) << "UpdateCloneInfo old record not exist"; + return -1; } return 0; } diff --git a/src/snapshotcloneserver/common/task.h b/src/snapshotcloneserver/common/task.h index bf69e03857..bc0faa4178 100644 --- a/src/snapshotcloneserver/common/task.h +++ b/src/snapshotcloneserver/common/task.h @@ -25,7 +25,7 @@ #include #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/task_tracker.h" namespace curve { diff --git a/src/snapshotcloneserver/common/task_tracker.h b/src/snapshotcloneserver/common/task_tracker.h index 8942f6bc26..872ebcacdc 100644 --- a/src/snapshotcloneserver/common/task_tracker.h +++ b/src/snapshotcloneserver/common/task_tracker.h @@ -28,7 +28,7 @@ #include #include "src/common/concurrent/concurrent.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" using ::curve::common::Mutex; using ::curve::common::ConditionVariable; diff --git a/src/snapshotcloneserver/snapshot/snapshot_core.cpp b/src/snapshotcloneserver/snapshot/snapshot_core.cpp index 97a1237d65..32b6e345c3 100644 --- a/src/snapshotcloneserver/snapshot/snapshot_core.cpp +++ b/src/snapshotcloneserver/snapshot/snapshot_core.cpp @@ -26,7 +26,7 @@ #include #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/snapshot/snapshot_task.h" #include "src/common/uuid.h" diff --git a/src/snapshotcloneserver/snapshot/snapshot_core.h b/src/snapshotcloneserver/snapshot/snapshot_core.h index 79639ed41e..3557291e89 100644 --- a/src/snapshotcloneserver/snapshot/snapshot_core.h +++ b/src/snapshotcloneserver/snapshot/snapshot_core.h @@ -31,7 +31,7 @@ #include "src/snapshotcloneserver/common/curvefs_client.h" #include "src/snapshotcloneserver/common/snapshotclone_meta_store.h" #include "src/snapshotcloneserver/snapshot/snapshot_data_store.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "src/snapshotcloneserver/common/snapshot_reference.h" #include "src/common/concurrent/name_lock.h" diff --git a/src/snapshotcloneserver/snapshot/snapshot_service_manager.h b/src/snapshotcloneserver/snapshot/snapshot_service_manager.h index ce2b51b038..1aa7143e9f 100644 --- a/src/snapshotcloneserver/snapshot/snapshot_service_manager.h +++ b/src/snapshotcloneserver/snapshot/snapshot_service_manager.h @@ -30,7 +30,7 @@ #include "src/snapshotcloneserver/snapshot/snapshot_core.h" #include "src/snapshotcloneserver/snapshot/snapshot_task.h" #include "src/snapshotcloneserver/snapshot/snapshot_task_manager.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "json/json.h" diff --git a/src/snapshotcloneserver/snapshot/snapshot_task.h b/src/snapshotcloneserver/snapshot/snapshot_task.h index b43c89bf6b..bf53993a61 100644 --- a/src/snapshotcloneserver/snapshot/snapshot_task.h +++ b/src/snapshotcloneserver/snapshot/snapshot_task.h @@ -28,7 +28,7 @@ #include #include "src/snapshotcloneserver/snapshot/snapshot_core.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/task.h" #include "src/snapshotcloneserver/common/task_info.h" #include "src/snapshotcloneserver/common/snapshotclone_metric.h" diff --git a/src/snapshotcloneserver/snapshot/snapshot_task_manager.cpp b/src/snapshotcloneserver/snapshot/snapshot_task_manager.cpp index 1ca2a27b79..9f61c067f5 100644 --- a/src/snapshotcloneserver/snapshot/snapshot_task_manager.cpp +++ b/src/snapshotcloneserver/snapshot/snapshot_task_manager.cpp @@ -21,7 +21,7 @@ */ #include "src/snapshotcloneserver/snapshot/snapshot_task_manager.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/common/concurrent/concurrent.h" diff --git a/src/snapshotcloneserver/snapshot/snapshot_task_manager.h b/src/snapshotcloneserver/snapshot/snapshot_task_manager.h index 006bde5842..a22eb0e2ae 100644 --- a/src/snapshotcloneserver/snapshot/snapshot_task_manager.h +++ b/src/snapshotcloneserver/snapshot/snapshot_task_manager.h @@ -33,7 +33,7 @@ #include "src/snapshotcloneserver/snapshot/snapshot_task.h" #include "src/snapshotcloneserver/common/thread_pool.h" #include "src/common/concurrent/rw_lock.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/config.h" #include "src/snapshotcloneserver/common/snapshotclone_metric.h" #include "src/snapshotcloneserver/snapshot/snapshot_core.h" diff --git a/src/snapshotcloneserver/snapshotclone_server.cpp b/src/snapshotcloneserver/snapshotclone_server.cpp index 869fc00711..314217be15 100644 --- a/src/snapshotcloneserver/snapshotclone_server.cpp +++ b/src/snapshotcloneserver/snapshotclone_server.cpp @@ -25,7 +25,7 @@ #include #include -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/snapshotclone_server.h" #include "src/common/curve_version.h" @@ -102,6 +102,10 @@ void InitSnapshotCloneServerOptions(std::shared_ptr conf, &serverOption->createCloneChunkConcurrency); conf->GetValueFatalIfFail("server.recoverChunkConcurrency", &serverOption->recoverChunkConcurrency); + conf->GetValueFatalIfFail("server.backEndReferenceRecordScanIntervalMs", + &serverOption->backEndReferenceRecordScanIntervalMs); + conf->GetValueFatalIfFail("server.backEndReferenceFuncScanIntervalMs", + &serverOption->backEndReferenceFuncScanIntervalMs); } void InitEtcdConf(std::shared_ptr conf, EtcdConf* etcdConf) { @@ -284,9 +288,13 @@ bool SnapShotCloneServer::Init() { } cloneTaskMgr_ = std::make_shared(cloneCore_, cloneMetric_); + + cloneServiceManagerBackend_ = + std::make_shared(cloneCore_); cloneServiceManager_ = std::make_shared( cloneTaskMgr_, - cloneCore_); + cloneCore_, + cloneServiceManagerBackend_); if (cloneServiceManager_->Init( snapshotCloneServerOptions_.serverOption) < 0) { LOG(ERROR) << "CloneServiceManager init fail."; diff --git a/src/snapshotcloneserver/snapshotclone_server.h b/src/snapshotcloneserver/snapshotclone_server.h index 8333295955..47163ddac4 100644 --- a/src/snapshotcloneserver/snapshotclone_server.h +++ b/src/snapshotcloneserver/snapshotclone_server.h @@ -33,7 +33,7 @@ #include "src/client/libcurve_file.h" #include "src/snapshotcloneserver/common/config.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/curvefs_client.h" #include "src/snapshotcloneserver/common/snapshotclone_meta_store.h" #include "src/snapshotcloneserver/common/snapshotclone_metric.h" @@ -149,6 +149,7 @@ class SnapShotCloneServer { std::shared_ptr cloneRef_; std::shared_ptr cloneCore_; std::shared_ptr cloneTaskMgr_; + std::shared_ptr cloneServiceManagerBackend_; std::shared_ptr cloneServiceManager_; std::shared_ptr service_; std::shared_ptr server_; diff --git a/src/snapshotcloneserver/snapshotclone_service.cpp b/src/snapshotcloneserver/snapshotclone_service.cpp index 28abce0b19..0a7e379d2b 100644 --- a/src/snapshotcloneserver/snapshotclone_service.cpp +++ b/src/snapshotcloneserver/snapshotclone_service.cpp @@ -27,7 +27,7 @@ #include #include "json/json.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/common/uuid.h" #include "src/common/string_util.h" #include "src/snapshotcloneserver/clone/clone_closure.h" @@ -82,6 +82,8 @@ void SnapshotCloneServiceImpl::default_method(RpcController* cntl, HandleGetFileSnapshotListAction(bcntl, requestId); } else if (*action == kGetCloneTaskListAction) { HandleGetCloneTaskListAction(bcntl, requestId); + } else if (*action == kGetCloneRefStatusAction) { + HandleGetCloneRefStatusAction(bcntl, requestId); } else { HandleBadRequestError(bcntl, requestId); } @@ -865,6 +867,73 @@ void SnapshotCloneServiceImpl::HandleGetCloneTaskListAction( return; } +void SnapshotCloneServiceImpl::HandleGetCloneRefStatusAction( + brpc::Controller* bcntl, const std::string &requestId) { + const std::string *version = + bcntl->http_request().uri().GetQuery(kVersionStr); + const std::string *user = + bcntl->http_request().uri().GetQuery(kUserStr); + const std::string *source = + bcntl->http_request().uri().GetQuery(kSourceStr); + if ((version == nullptr) || + (user == nullptr) || + (source == nullptr) || + (version->empty()) || + (source->empty()) || + (user->empty())) { + HandleBadRequestError(bcntl, requestId); + return; + } + + LOG(INFO) << "GetCloneRefStatus:" + << " Version = " << *version + << ", User = " << *user + << ", Source = " << *source + << ", requestId = " << requestId; + + std::vector cloneInfos; + CloneRefStatus refStatus; + int ret = cloneManager_->GetCloneRefStatus(*source, &refStatus, + &cloneInfos); + if (ret < 0) { + bcntl->http_response().set_status_code( + brpc::HTTP_STATUS_INTERNAL_SERVER_ERROR); + SetErrorMessage(bcntl, ret, requestId); + return; + } + + bcntl->http_response().set_status_code(brpc::HTTP_STATUS_OK); + butil::IOBufBuilder os; + Json::Value mainObj; + mainObj[kCodeStr] = std::to_string(kErrCodeSuccess); + mainObj[kMessageStr] = code2Msg[kErrCodeSuccess]; + mainObj[kRequestIdStr] = requestId; + mainObj[kRefStatusStr] = static_cast (refStatus); + mainObj[kTotalCountStr] = 0; + if (refStatus == CloneRefStatus::kNeedCheck) { + mainObj[kTotalCountStr] = cloneInfos.size(); + Json::Value listObj; + for (int i = 0; i < cloneInfos.size(); i++) { + Json::Value cloneTaskObj; + cloneTaskObj[kUserStr] = cloneInfos[i].GetUser(); + cloneTaskObj[kFileStr] = cloneInfos[i].GetDest(); + if (cloneInfos[i].GetTaskType() == CloneTaskType::kClone) { + cloneTaskObj[kInodeStr] = cloneInfos[i].GetDestId(); + } else { + cloneTaskObj[kInodeStr] = cloneInfos[i].GetOriginId(); + } + + listObj.append(cloneTaskObj); + } + + mainObj[kCloneFileInfoStr] = listObj; + } + + os << mainObj.toStyledString(); + os.move_to(bcntl->response_attachment()); + return; +} + void SnapshotCloneServiceImpl::SetErrorMessage(brpc::Controller* bcntl, int errCode, const std::string &requestId, diff --git a/src/snapshotcloneserver/snapshotclone_service.h b/src/snapshotcloneserver/snapshotclone_service.h index 3005913328..6ba1f34f48 100644 --- a/src/snapshotcloneserver/snapshotclone_service.h +++ b/src/snapshotcloneserver/snapshotclone_service.h @@ -92,6 +92,8 @@ class SnapshotCloneServiceImpl : public SnapshotCloneService { const std::string &requestId); void HandleGetCloneTaskListAction(brpc::Controller* bcntl, const std::string &requestId); + void HandleGetCloneRefStatusAction(brpc::Controller* bcntl, + const std::string &requestId); bool CheckBoolParamter( const std::string *param, bool *valueOut); void SetErrorMessage(brpc::Controller* bcntl, int errCode, diff --git a/test/chunkserver/heartbeat_test_common.h b/test/chunkserver/heartbeat_test_common.h index 02529be1f2..6dabe90eba 100644 --- a/test/chunkserver/heartbeat_test_common.h +++ b/test/chunkserver/heartbeat_test_common.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include //NOLINT #include "include/chunkserver/chunkserver_common.h" #include "src/chunkserver/copyset_node_manager.h" diff --git a/test/integration/heartbeat/common.h b/test/integration/heartbeat/common.h index 030983e9e7..f38a4d139b 100644 --- a/test/integration/heartbeat/common.h +++ b/test/integration/heartbeat/common.h @@ -29,8 +29,8 @@ #include #include -#include -#include +#include //NOLINT +#include //NOLINT #include #include #include diff --git a/test/integration/snapshotcloneserver/fake_snapshotclone_meta_store.cpp b/test/integration/snapshotcloneserver/fake_snapshotclone_meta_store.cpp index f9ce48c028..7b2f37b937 100644 --- a/test/integration/snapshotcloneserver/fake_snapshotclone_meta_store.cpp +++ b/test/integration/snapshotcloneserver/fake_snapshotclone_meta_store.cpp @@ -57,7 +57,7 @@ int FakeSnapshotCloneMetaStore::UpdateSnapshot(const SnapshotInfo &info) { if (search != snapInfos_.end()) { search->second = info; } else { - snapInfos_.emplace(info.GetUuid(), info); + return -1; } return 0; } @@ -129,7 +129,7 @@ int FakeSnapshotCloneMetaStore::UpdateCloneInfo(const CloneInfo &info) { if (search != cloneInfos_.end()) { search->second = info; } else { - cloneInfos_.emplace(info.GetTaskId(), info); + return -1; } return 0; } diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp index 7b49cccc51..7914945f58 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp @@ -30,7 +30,7 @@ #include "src/snapshotcloneserver/snapshot/snapshot_service_manager.h" #include "src/snapshotcloneserver/clone/clone_service_manager.h" #include "test/integration/snapshotcloneserver/test_snapshotcloneserver_helpler.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/snapshotclone_meta_store.h" using curve::CurveCluster; @@ -91,6 +91,7 @@ const std::vector mdsConfigOptions{ std::string("mds.DbName=") + kMdsDbName, std::string("mds.file.expiredTimeUs=50000"), std::string("mds.file.expiredTimeUs=10000"), + std::string("mds.snapshotcloneclient.addr=") + kSnapshotCloneServerIpPort, }; const std::vector mdsConf1{ @@ -197,6 +198,8 @@ const std::vector snapshotcloneserverConfigOptions{ std::string("server.dummy.listen.port=") + kSnapshotCloneServerDummyServerPort, std::string("leader.campagin.prefix=") + kLeaderCampaginPrefix, + std::string("server.backEndReferenceRecordScanIntervalMs=100"), + std::string("server.backEndReferenceFuncScanIntervalMs=1000"), }; const std::vector snapshotcloneConf{ @@ -215,6 +218,7 @@ const char* testFile1_ = "/ItUser1/file1"; const char* testFile2_ = "/ItUser1/file2"; const char* testFile3_ = "/ItUser2/file3"; const char* testFile4_ = "/ItUser1/file3"; +const char* testFile5_ = "/ItUser1/file4"; const char* testUser1_ = "ItUser1"; const char* testUser2_ = "ItUser2"; @@ -356,6 +360,9 @@ class SnapshotCloneServerTest : public ::testing::Test { ASSERT_EQ(0, fileClient_->Create(testFile4_, userinfo, 10ULL * 1024 * 1024 * 1024)); + + ASSERT_EQ(0, fileClient_->Create(testFile5_, userinfo, + 10ULL * 1024 * 1024 * 1024)); } static bool CreateAndWriteFile(const std::string& fileName, @@ -1049,5 +1056,61 @@ TEST_F(SnapshotCloneServerTest, TestCloneHasSameDest) { ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); } +// lazy克隆卷，删除克隆卷，再删除源卷，源卷需要可以删除 +TEST_F(SnapshotCloneServerTest, TestDeleteLazyCloneDestThenDeleteSrc) { + // 操作1：testUser1_ clone 镜像testFile5_，lazy克隆两个卷dstFile1，dstFile2 + // 预期1 返回克隆成功 + std::string uuid1; + std::string uuid2; + std::string dstFile1 = "/dest1"; + std::string dstFile2 = "/dest2"; + UserInfo_t userinfo; + userinfo.owner = testUser1_; + int ret = + CloneOrRecover("Clone", testUser1_, testFile5_, dstFile1, true, &uuid1); + ASSERT_EQ(0, ret); + + ret = + CloneOrRecover("Clone", testUser1_, testFile5_, dstFile2, true, &uuid2); + ASSERT_EQ(0, ret); + + // 删除源卷，删除失败，卷被占用 + + ret = fileClient_->Unlink(testFile5_, userinfo, false); + ASSERT_EQ(-27, ret); + + // 操作2：删除目的卷dstFile1成功，再次删除源卷 + // 预期2 删除失败，卷被占用 + ret = fileClient_->Unlink(dstFile1, userinfo, false); + ASSERT_EQ(0, ret); + + ret = fileClient_->Unlink(testFile5_, userinfo, false); + ASSERT_EQ(-27, ret); + + + // 操作3：删除目的卷dstFile2成功，再次删除源卷 + // 预期3 删除成功 + ret = fileClient_->Unlink(dstFile2, userinfo, false); + ASSERT_EQ(0, ret); + + ret = fileClient_->Unlink(testFile5_, userinfo, false); + ASSERT_EQ(0, ret); + + // 操作4： 等待一段时间，看垃圾记录后台能否删除 + bool noRecord = false; + for (int i = 0; i < 100; i++) { + TaskCloneInfo info; + int ret1 = GetCloneTaskInfo(testUser1_, uuid1, &info); + int ret2 = GetCloneTaskInfo(testUser1_, uuid2, &info); + if (ret1 == kErrCodeFileNotExist && ret2 == kErrCodeFileNotExist) { + noRecord = true; + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(3000)); + } + + ASSERT_TRUE(noRecord); +} } // namespace snapshotcloneserver } // namespace curve diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp index bad4111889..cc0a30808f 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp @@ -30,7 +30,7 @@ #include "src/snapshotcloneserver/snapshot/snapshot_service_manager.h" #include "src/snapshotcloneserver/clone/clone_service_manager.h" #include "test/integration/snapshotcloneserver/test_snapshotcloneserver_helpler.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/snapshotclone_meta_store.h" using curve::CurveCluster; @@ -90,6 +90,7 @@ const std::vector mdsConfigOptions{ std::string("mds.DbName=") + kMdsDbName, std::string("mds.file.expiredTimeUs=50000"), std::string("mds.file.expiredTimeUs=10000"), + std::string("mds.snapshotcloneclient.addr=") + kSnapshotCloneServerIpPort, }; const std::vector mdsConf1{ @@ -198,6 +199,8 @@ const std::vector snapshotcloneserverConfigOptions{ std::string("server.dummy.listen.port=") + kSnapshotCloneServerDummyServerPort, std::string("leader.campagin.prefix=") + kLeaderCampaginPrefix, + std::string("server.backEndReferenceRecordScanIntervalMs=100"), + std::string("server.backEndReferenceFuncScanIntervalMs=1000"), }; const std::vector snapshotcloneConf{ diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_exception_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_exception_test.cpp index f662fb9c1f..b7097d76f2 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_exception_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_exception_test.cpp @@ -56,6 +56,8 @@ class SnapshotCloneServerTest : public ::testing::Test { options_->createCloneChunkConcurrency = 8; options_->recoverChunkConcurrency = 8; options_->clientAsyncMethodRetryTimeSec = 1; + options_->backEndReferenceRecordScanIntervalMs = 100; + options_->backEndReferenceFuncScanIntervalMs = 1000; server_ = new SnapshotCloneServerModule(); server_->Start(*options_); diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_module.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_module.cpp index 5423ba0e52..2e549688b8 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_module.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_module.cpp @@ -86,9 +86,12 @@ int SnapshotCloneServerModule::Start( std::shared_ptr cloneTaskMgr = std::make_shared(cloneCore, cloneMetric); + auto cloneServiceManagerBackend = + std::make_shared(cloneCore); + cloneServiceManager_ = std::make_shared(cloneTaskMgr, - cloneCore); + cloneCore, cloneServiceManagerBackend); if (cloneServiceManager_->Init(serverOption_) < 0) { LOG(ERROR) << "CloneServiceManager init fail."; return kErrCodeServerInitFail; diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_module.h b/test/integration/snapshotcloneserver/snapshotcloneserver_module.h index e15a18ffe2..8b5d4dca33 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_module.h +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_module.h @@ -35,7 +35,7 @@ #include "src/snapshotcloneserver/clone/clone_service_manager.h" #include "src/common/configuration.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/curvefs_client.h" #include "src/snapshotcloneserver/snapshot/snapshot_data_store.h" diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp index f65fa184a0..cba89d1922 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp @@ -32,7 +32,7 @@ #include "src/snapshotcloneserver/snapshot/snapshot_service_manager.h" #include "src/snapshotcloneserver/clone/clone_service_manager.h" #include "test/integration/snapshotcloneserver/test_snapshotcloneserver_helpler.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/common/snapshotclone_meta_store.h" using curve::CurveCluster; @@ -114,6 +114,7 @@ const std::vector mdsConfigOptions{ "mds.topology.TopologyUpdateToRepoSec=5", std::string("mds.file.expiredTimeUs=50000"), std::string("mds.file.expiredTimeUs=10000"), + std::string("mds.snapshotcloneclient.addr=") + kSnapshotCloneServerIpPort, }; const std::vector mdsConf1{ @@ -224,6 +225,8 @@ const std::vector snapshotcloneserverConfigOptions{ std::string("server.dummy.listen.port=") + kSnapshotCloneServerDummyServerPort, std::string("leader.campagin.prefix=") + kLeaderCampaginPrefix, + std::string("server.backEndReferenceRecordScanIntervalMs=100"), + std::string("server.backEndReferenceFuncScanIntervalMs=1000"), }; const std::vector snapshotcloneConf{ diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp index df6cd625c8..b1fe105d3b 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp @@ -77,6 +77,7 @@ const std::vector mdsConfigOptions{ "mds.enable.replica.scheduler=false", "mds.heartbeat.misstimeoutMs=10000", "mds.topology.TopologyUpdateToRepoSec=5", + std::string("mds.snapshotcloneclient.addr=") + kSnapshotCloneServerIpPort, }; const std::vector mdsConf1{ @@ -111,6 +112,8 @@ const std::vector snapshotcloneserverConfigOptions{ kSnapshotCloneServerDummyServerPort, std::string("leader.campagin.prefix=") + kLeaderCampaginPrefix, std::string("server.address=") + kSnapshotCloneServerIpPort, + std::string("server.backEndReferenceRecordScanIntervalMs=100"), + std::string("server.backEndReferenceFuncScanIntervalMs=1000"), }; const std::vector snapshotcloneConf{ diff --git a/test/mds/mock/BUILD b/test/mds/mock/BUILD index f341b4195d..f78164428f 100644 --- a/test/mds/mock/BUILD +++ b/test/mds/mock/BUILD @@ -27,6 +27,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//proto:chunkserver-cc-protos", + "//proto:snapshotcloneserver_cc_proto", "//src/mds/nameserver2" ], ) diff --git a/test/mds/mock/mock_snapshotcloneserver.h b/test/mds/mock/mock_snapshotcloneserver.h new file mode 100644 index 0000000000..6f7c7963d7 --- /dev/null +++ b/test/mds/mock/mock_snapshotcloneserver.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2020/12/2 + * Author: hzchenwei7 + */ + +#ifndef TEST_MDS_MOCK_MOCK_SNAPSHOTCLONESERVER_H_ +#define TEST_MDS_MOCK_MOCK_SNAPSHOTCLONESERVER_H_ + +#include "proto/snapshotcloneserver.pb.h" + +namespace curve { +namespace snapshotcloneserver { + +using ::google::protobuf::RpcController; +using ::google::protobuf::Closure; + +class MockSnapshotCloneService : public SnapshotCloneService { + public: + MOCK_METHOD4(default_method, + void(RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done)); +}; + +} // namespace snapshotcloneserver +} // namespace curve + + +#endif // TEST_MDS_MOCK_MOCK_SNAPSHOTCLONESERVER_H_ diff --git a/test/mds/nameserver2/curvefs_test.cpp b/test/mds/nameserver2/curvefs_test.cpp index 2a0378bb2a..e7f088733b 100644 --- a/test/mds/nameserver2/curvefs_test.cpp +++ b/test/mds/nameserver2/curvefs_test.cpp @@ -32,6 +32,7 @@ #include "test/mds/nameserver2/mock/mock_inode_id_generator.h" #include "test/mds/nameserver2/mock/mock_chunk_allocate.h" #include "test/mds/nameserver2/mock/mock_clean_manager.h" +#include "test/mds/nameserver2/mock/mock_snapshotclone_client.h" #include "test/mds/mock/mock_alloc_statistic.h" #include "test/mds/mock/mock_topology.h" @@ -47,6 +48,8 @@ using curve::common::Authenticator; using curve::common::TimeUtility; using curve::mds::topology::MockTopology; +using curve::mds::snapshotcloneclient::MockSnapshotCloneClient; +using curve::mds::snapshotcloneclient::DestFileInfo; namespace curve { namespace mds { @@ -61,7 +64,7 @@ class CurveFSTest: public ::testing::Test { mockcleanManager_ = std::make_shared(); topology_ = std::make_shared(); fileRecordManager_ = std::make_shared(); - + snapshotClient_ = std::make_shared(); // session repo已经mock，数据库相关参数不需要 fileRecordOptions_.fileRecordExpiredTimeUs = 5 * 1000; fileRecordOptions_.scanIntervalTimeUs = 1 * 1000; @@ -92,7 +95,8 @@ class CurveFSTest: public ::testing::Test { fileRecordManager_, allocStatistic_, curveFSOptions_, - topology_); + topology_, + snapshotClient_); curvefs_->Run(); } @@ -109,6 +113,7 @@ class CurveFSTest: public ::testing::Test { std::shared_ptr fileRecordManager_; std::shared_ptr allocStatistic_; std::shared_ptr topology_; + std::shared_ptr snapshotClient_; struct FileRecordOptions fileRecordOptions_; struct RootAuthOption authOptions_; struct CurveFSOption curveFSOptions_; @@ -390,6 +395,10 @@ TEST_F(CurveFSTest, testDeleteFile) { .WillOnce(DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, PutFile(_)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + std::vector fileInfoList; EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) .Times(1) @@ -438,6 +447,10 @@ TEST_F(CurveFSTest, testDeleteFile) { .WillOnce(DoAll(SetArgPointee<2>(fileInfoList), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, PutFile(_)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + EXPECT_CALL(*mockcleanManager_, GetTask(_)) .Times(1) @@ -577,6 +590,230 @@ TEST_F(CurveFSTest, testDeleteFile) { ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), StatusCode::kNotSupported); } + + // test delete pagefile, file under clone + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kHasRef; + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + Return(StatusCode::kOK))); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kDeleteFileBeingCloned); + } + + // test delete pagefile, file under clone but has no ref but delete fail + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kNoRef; + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + Return(StatusCode::kOK))); + + EXPECT_CALL(*storage_, MoveFileToRecycle(_, _)) + .Times(1) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kStorageError); + } + + // test delete pagefile, file under clone but has no ref success + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kNoRef; + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + Return(StatusCode::kOK))); + + EXPECT_CALL(*storage_, MoveFileToRecycle(_, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kOK); + } + + // test delete pagefile, file under clone but need check list empty + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kNeedCheck; + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + Return(StatusCode::kOK))); + + EXPECT_CALL(*storage_, MoveFileToRecycle(_, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kOK); + } + + // test delete pagefile, file under clone but need check, file has ref + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::KeyNotExist))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kNeedCheck; + std::vector fileCheckList; + DestFileInfo info; + info.filename = "/file"; + info.inodeid = 100; + fileCheckList.push_back(info); + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + SetArgPointee<3>(fileCheckList), + Return(StatusCode::kOK))); + + EXPECT_CALL(*storage_, MoveFileToRecycle(_, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kOK); + } + + // test delete pagefile, file under clone but need check, inode mismatch + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + fileInfo.set_id(10); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kNeedCheck; + std::vector fileCheckList; + DestFileInfo info; + info.filename = "/file"; + info.inodeid = 100; + fileCheckList.push_back(info); + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + SetArgPointee<3>(fileCheckList), + Return(StatusCode::kOK))); + + EXPECT_CALL(*storage_, MoveFileToRecycle(_, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kOK); + } + + // test delete pagefile, file under clone but need check, has ref + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + fileInfo.set_id(100); + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))) + .WillOnce(DoAll(SetArgPointee<2>(fileInfo), + Return(StoreStatus::OK))); + + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .Times(1) + .WillOnce(Return(StoreStatus::OK)); + + CloneRefStatus status = CloneRefStatus::kNeedCheck; + std::vector fileCheckList; + DestFileInfo info; + info.filename = "/file"; + info.inodeid = 100; + fileCheckList.push_back(info); + EXPECT_CALL(*snapshotClient_, GetCloneRefStatus(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(status), + SetArgPointee<3>(fileCheckList), + Return(StatusCode::kOK))); + + // EXPECT_CALL(*storage_, MoveFileToRecycle(_, _)) + // .Times(1) + // .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(curvefs_->DeleteFile("/file1", kUnitializedFileID, false), + StatusCode::kDeleteFileBeingCloned); + } } TEST_F(CurveFSTest, testGetAllocatedSize) { @@ -3086,7 +3323,8 @@ TEST_F(CurveFSTest, Init) { fileRecordManager_, allocStatistic_, curveFSOptions_, - topology_)); + topology_, + nullptr)); } } @@ -3103,7 +3341,8 @@ TEST_F(CurveFSTest, Init) { fileRecordManager_, allocStatistic_, curveFSOptions_, - topology_)); + topology_, + nullptr)); } // test getfile not exist @@ -3124,7 +3363,8 @@ TEST_F(CurveFSTest, Init) { fileRecordManager_, allocStatistic_, curveFSOptions_, - topology_)); + topology_, + nullptr)); // putfile ok FileInfo fileInfo5; @@ -3149,7 +3389,8 @@ TEST_F(CurveFSTest, Init) { fileRecordManager_, allocStatistic_, curveFSOptions_, - topology_)); + topology_, + nullptr)); } } diff --git a/test/mds/nameserver2/file_record_test.cpp b/test/mds/nameserver2/file_record_test.cpp index 730208d7d9..7d3015ade5 100644 --- a/test/mds/nameserver2/file_record_test.cpp +++ b/test/mds/nameserver2/file_record_test.cpp @@ -23,8 +23,8 @@ #include #include -#include -#include +#include //NOLINT +#include // NOLINT #include "src/common/timeutility.h" #include "src/mds/common/mds_define.h" diff --git a/test/mds/nameserver2/mock/BUILD b/test/mds/nameserver2/mock/BUILD index f051a5d7d7..b458c48592 100644 --- a/test/mds/nameserver2/mock/BUILD +++ b/test/mds/nameserver2/mock/BUILD @@ -20,6 +20,7 @@ cc_library( deps = [ "//src/mds/topology:topology", "//src/mds/nameserver2:nameserver2", + "//src/mds/snapshotcloneclient:snapshotcloneclient" ], visibility = ["//visibility:public"], ) diff --git a/test/mds/nameserver2/mock/mock_snapshotclone_client.h b/test/mds/nameserver2/mock/mock_snapshotclone_client.h new file mode 100644 index 0000000000..7a61ef700c --- /dev/null +++ b/test/mds/nameserver2/mock/mock_snapshotclone_client.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2020/12/03 + * Author: hzchenwei7 + */ + +#ifndef TEST_MDS_NAMESERVER2_MOCK_MOCK_SNAPSHOTCLONE_CLIENT_H_ +#define TEST_MDS_NAMESERVER2_MOCK_MOCK_SNAPSHOTCLONE_CLIENT_H_ + +#include +#include +#include +#include "src/mds/snapshotcloneclient/snapshotclone_client.h" + +namespace curve { +namespace mds { +namespace snapshotcloneclient { +class MockSnapshotCloneClient: public SnapshotCloneClient { + public: + ~MockSnapshotCloneClient() {} + MOCK_METHOD1(Init, void(const SnapshotCloneClientOption &option)); + + MOCK_METHOD4(GetCloneRefStatus, StatusCode(std::string, std::string, + CloneRefStatus *, std::vector *)); +}; +} // namespace snapshotcloneclient +} // namespace mds +} // namespace curve +#endif // TEST_MDS_NAMESERVER2_MOCK_MOCK_SNAPSHOTCLONE_CLIENT_H_ diff --git a/test/mds/nameserver2/namespace_service_test.cpp b/test/mds/nameserver2/namespace_service_test.cpp index 11befe1b64..93b0c5fcdd 100644 --- a/test/mds/nameserver2/namespace_service_test.cpp +++ b/test/mds/nameserver2/namespace_service_test.cpp @@ -102,7 +102,8 @@ class NameSpaceServiceTest : public ::testing::Test { cleanManager_, fileRecordManager_, allocStatistic_, - curveFSOptions, topology_); + curveFSOptions, topology_, + nullptr); kCurveFS.Run(); std::this_thread::sleep_for(std::chrono::microseconds( diff --git a/test/mds/snapshotcloneclient/BUILD b/test/mds/snapshotcloneclient/BUILD new file mode 100644 index 0000000000..ba708a6954 --- /dev/null +++ b/test/mds/snapshotcloneclient/BUILD @@ -0,0 +1,33 @@ +# +# Copyright (c) 2020 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load( + "//:copts.bzl", + "GCC_FLAGS", + "GCC_TEST_FLAGS", +) + +cc_test( + name = "snapshotcloneclient_utest", + srcs = glob(["*.cpp", "*.h"]), + copts = GCC_TEST_FLAGS, + deps = [ + "//external:gtest", + "//src/mds/snapshotcloneclient:snapshotcloneclient", + "//test/mds/mock:common_mock", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/test/mds/snapshotcloneclient/test_snapshotclone_client.cpp b/test/mds/snapshotcloneclient/test_snapshotclone_client.cpp new file mode 100644 index 0000000000..d2f62b6d9e --- /dev/null +++ b/test/mds/snapshotcloneclient/test_snapshotclone_client.cpp @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2020/12/02 + * Author: hzchenwei7 + */ + + +#include +#include +// #include +// #include +#include +#include "src/mds/snapshotcloneclient/snapshotclone_client.h" +#include "test/mds/mock/mock_snapshotcloneserver.h" +#include "json/json.h" +#include "src/common/snapshotclone/snapshotclone_define.h" + +using ::curve::snapshotcloneserver::MockSnapshotCloneService; +using ::curve::snapshotcloneserver::HttpRequest; +using ::curve::snapshotcloneserver::HttpResponse; +using ::google::protobuf::RpcController; +using ::google::protobuf::Closure; +using ::curve::snapshotcloneserver::kCodeStr; +using ::curve::snapshotcloneserver::kErrCodeSuccess; +using ::curve::snapshotcloneserver::kErrCodeInternalError; +using ::curve::snapshotcloneserver::kRefStatusStr; +using ::curve::snapshotcloneserver::CloneRefStatus; + +using ::testing::Return; +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::Invoke; + +namespace curve { +namespace mds { +namespace snapshotcloneclient { + +class TestSnapshotCloneClient : public ::testing::Test { + protected: + TestSnapshotCloneClient() {} + void SetUp() { + server_ = new brpc::Server(); + client_ = std::make_shared(); + + mocksnapshotcleonservice_ = new MockSnapshotCloneService(); + ASSERT_EQ(server_->AddService(mocksnapshotcleonservice_, + brpc::SERVER_DOESNT_OWN_SERVICE), 0); + ASSERT_EQ(0, server_->Start("127.0.0.1", {8900, 8999}, nullptr)); + listenAddr_ = server_->listen_address(); + } + void TearDown() { + client_ = nullptr; + server_->Stop(0); + server_->Join(); + delete server_; + server_ = nullptr; + delete mocksnapshotcleonservice_; + mocksnapshotcleonservice_ = nullptr; + } + + protected: + std::shared_ptr client_; + butil::EndPoint listenAddr_; + brpc::Server *server_; + MockSnapshotCloneService *mocksnapshotcleonservice_; + SnapshotCloneClientOption option; +}; + +TEST_F(TestSnapshotCloneClient, TestInitSuccess) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); +} + +TEST_F(TestSnapshotCloneClient, TestInitFalse) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = ""; + client_->Init(option); + ASSERT_FALSE(client_->GetInitStatus()); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusFalseNotInit) { + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::kSnapshotCloneServerNotInit); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusFalseConnectFail) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "aa"; + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::kSnapshotCloneConnectFail); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusFalseCallFail) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(0); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::KInternalError); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusFalseParseFail) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + + HttpResponse response; + EXPECT_CALL(*mocksnapshotcleonservice_, default_method(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke([](RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done){ + brpc::ClosureGuard doneGuard(done); + }))); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::KInternalError); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusFalseRetNot0) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + + HttpResponse response; + EXPECT_CALL(*mocksnapshotcleonservice_, default_method(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke([](RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done){ + brpc::Controller* bcntl = + static_cast(controller); + bcntl->http_response().set_status_code( + brpc::HTTP_STATUS_OK); + butil::IOBufBuilder os; + Json::Value mainObj; + mainObj[kCodeStr] = std::to_string( + kErrCodeInternalError); + os << mainObj.toStyledString(); + os.move_to(bcntl->response_attachment()); + brpc::ClosureGuard doneGuard(done); + }))); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::KInternalError); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusFalseInvalidStatus) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + + HttpResponse response; + EXPECT_CALL(*mocksnapshotcleonservice_, default_method(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke([](RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done){ + brpc::Controller* bcntl = + static_cast(controller); + bcntl->http_response().set_status_code( + brpc::HTTP_STATUS_OK); + butil::IOBufBuilder os; + Json::Value mainObj; + mainObj[kCodeStr] = std::to_string(kErrCodeSuccess); + CloneRefStatus refStatus = CloneRefStatus::kNoRef; + mainObj[kRefStatusStr] = 4; + os << mainObj.toStyledString(); + os.move_to(bcntl->response_attachment()); + brpc::ClosureGuard doneGuard(done); + }))); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::KInternalError); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusSuccessNoRef) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + + HttpResponse response; + EXPECT_CALL(*mocksnapshotcleonservice_, default_method(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke([](RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done){ + brpc::Controller* bcntl = + static_cast(controller); + bcntl->http_response().set_status_code( + brpc::HTTP_STATUS_OK); + butil::IOBufBuilder os; + Json::Value mainObj; + mainObj[kCodeStr] = std::to_string(kErrCodeSuccess); + CloneRefStatus refStatus = CloneRefStatus::kNoRef; + mainObj[kRefStatusStr] = static_cast (refStatus); + os << mainObj.toStyledString(); + os.move_to(bcntl->response_attachment()); + brpc::ClosureGuard doneGuard(done); + }))); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::kOK); + ASSERT_EQ(status, CloneRefStatus::kNoRef); + ASSERT_EQ(0, fileCheckList.size()); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusSuccessHasRef) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + CloneRefStatus refStatus = CloneRefStatus::kHasRef; + HttpResponse response; + EXPECT_CALL(*mocksnapshotcleonservice_, default_method(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke([refStatus](RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done){ + brpc::Controller* bcntl = + static_cast(controller); + bcntl->http_response().set_status_code( + brpc::HTTP_STATUS_OK); + butil::IOBufBuilder os; + Json::Value mainObj; + mainObj[kCodeStr] = std::to_string(kErrCodeSuccess); + mainObj[kRefStatusStr] = static_cast (refStatus); + os << mainObj.toStyledString(); + os.move_to(bcntl->response_attachment()); + brpc::ClosureGuard doneGuard(done); + }))); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::kOK); + ASSERT_EQ(status, refStatus); + ASSERT_EQ(0, fileCheckList.size()); +} + +TEST_F(TestSnapshotCloneClient, TestGetCloneRefStatusSuccessNeedCheck) { + uint32_t port = listenAddr_.port; + option.snapshotCloneAddr = "127.0.0.1:" + std::to_string(port); + client_->Init(option); + ASSERT_TRUE(client_->GetInitStatus()); + CloneRefStatus refStatus = CloneRefStatus::kNeedCheck; + HttpResponse response; + EXPECT_CALL(*mocksnapshotcleonservice_, default_method(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke([refStatus](RpcController *controller, + const HttpRequest *request, + HttpResponse *response, + Closure *done){ + brpc::Controller* bcntl = + static_cast(controller); + bcntl->http_response().set_status_code( + brpc::HTTP_STATUS_OK); + butil::IOBufBuilder os; + Json::Value mainObj; + mainObj[kCodeStr] = std::to_string(kErrCodeSuccess); + mainObj[kRefStatusStr] = static_cast (refStatus); + mainObj["TotalCount"] = 1; + Json::Value listObj; + Json::Value cloneTaskObj; + cloneTaskObj["User"] = "user"; + cloneTaskObj["File"] = "/dest"; + cloneTaskObj["Inode"] = 100; + listObj.append(cloneTaskObj); + mainObj["CloneFileInfo"] = listObj; + os << mainObj.toStyledString(); + os.move_to(bcntl->response_attachment()); + brpc::ClosureGuard doneGuard(done); + }))); + + std::string filename = "/file"; + std::string user = "test"; + CloneRefStatus status; + std::vector fileCheckList; + auto ret = client_->GetCloneRefStatus(filename, user, + &status, &fileCheckList); + ASSERT_EQ(ret, StatusCode::kOK); + ASSERT_EQ(status, refStatus); + ASSERT_EQ(1, fileCheckList.size()); +} + +} // namespace snapshotcloneclient +} // namespace mds +} // namespace curve + diff --git a/test/snapshotcloneserver/mock_snapshot_server.h b/test/snapshotcloneserver/mock_snapshot_server.h index 758a478a0f..4d2272592c 100644 --- a/test/snapshotcloneserver/mock_snapshot_server.h +++ b/test/snapshotcloneserver/mock_snapshot_server.h @@ -298,7 +298,7 @@ class MockSnapshotServiceManager : public SnapshotServiceManager { class MockCloneServiceManager : public CloneServiceManager { public: MockCloneServiceManager() : - CloneServiceManager(nullptr, nullptr) {} + CloneServiceManager(nullptr, nullptr, nullptr) {} ~MockCloneServiceManager() {} MOCK_METHOD6(CloneFile, @@ -345,6 +345,11 @@ class MockCloneServiceManager : public CloneServiceManager { MOCK_METHOD2(CleanCloneTask, int(const std::string &user, const TaskIdType &taskId)); + + MOCK_METHOD3(GetCloneRefStatus, + int(const std::string &src, + CloneRefStatus *refStatus, + std::vector *needCheckFiles)); }; class MockCloneCore : public CloneCore { @@ -390,6 +395,20 @@ class MockCloneCore : public CloneCore { MOCK_METHOD1(HandleRemoveCloneOrRecoverTask, int(std::shared_ptr task)); + + MOCK_METHOD2(CheckFileExists, + int(const std::string &filename, uint64_t inodeId)); + + MOCK_METHOD1(HandleDeleteCloneInfo, + int(const CloneInfo &cloneInfo)); +}; + +class MockCloneServiceManagerBackend : public CloneServiceManagerBackend { + public: + MOCK_METHOD0(Func, void()); + MOCK_METHOD2(Init, void(uint32_t recordIntevalMs, uint32_t roundIntevalMs)); + MOCK_METHOD0(Start, void()); + MOCK_METHOD0(Stop, void()); }; class MockKVStorageClient : public KVStorageClient { diff --git a/test/snapshotcloneserver/test_client.conf b/test/snapshotcloneserver/test_client.conf new file mode 100644 index 0000000000..fcc42f0fa0 --- /dev/null +++ b/test/snapshotcloneserver/test_client.conf @@ -0,0 +1,32 @@ +chunkserver.checkHealthTimeoutMs=100 +chunkserver.enableAppliedIndexRead=1 +chunkserver.maxRPCTimeoutMS=1 +chunkserver.maxRetrySleepIntervalUS=1 +chunkserver.maxRetryTimesBeforeConsiderSuspend=20 +chunkserver.maxStableTimeoutTimes=10 +chunkserver.minRetryTimesForceTimeoutBackoff=5 +chunkserver.opMaxRetry=1 +chunkserver.opRetryIntervalUS=1 +chunkserver.rpcTimeoutMS=1 +chunkserver.serverStableThreshold=3 +global.fileIOSplitMaxSizeKB=64 +global.fileMaxInFlightRPCNum=128 +global.logLevel=0 +global.logPath=/data/log/curve/ +global.metricDummyServerStartPort=9000 +global.turnOffHealthCheck=true +isolation.taskQueueCapacity=1000000 +isolation.taskThreadPoolSize=1 +mds.listen.addr=127.0.0.1:8888 +mds.maxFailedTimesBeforeChangeMDS=2 +mds.maxRPCTimeoutMS=1 +mds.maxRetryMS=1 +mds.refreshTimesPerLease=4 +mds.registerToMDS=false +mds.rpcRetryIntervalUS=1 +mds.rpcTimeoutMS=1 +metacache.getLeaderRetry=1 +metacache.getLeaderTimeOutMS=1 +metacache.rpcRetryIntervalUS=1 +schedule.queueCapacity=1000000 +schedule.threadpoolSize=2 diff --git a/test/snapshotcloneserver/test_clone_core.cpp b/test/snapshotcloneserver/test_clone_core.cpp index e0ddd4ac3d..a6338d64d5 100644 --- a/test/snapshotcloneserver/test_clone_core.cpp +++ b/test/snapshotcloneserver/test_clone_core.cpp @@ -25,7 +25,7 @@ #include "src/snapshotcloneserver/clone/clone_core.h" #include "src/snapshotcloneserver/clone/clone_task.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/common/location_operator.h" #include "test/snapshotcloneserver/mock_snapshot_server.h" @@ -1628,6 +1628,115 @@ TEST_F(TestCloneCoreImpl, core_->HandleCleanCloneOrRecoverTask(task); } +TEST_F(TestCloneCoreImpl, + TestCheckFileExists) { + FInfo fInfo; + fInfo.id = 100; + EXPECT_CALL(*client_, GetFileInfo(_, _, _)) + .WillOnce(DoAll( + SetArgPointee<2>(fInfo), + Return(LIBCURVE_ERROR::OK))); + + ASSERT_EQ(core_->CheckFileExists("filename", 100), kErrCodeFileExist); + + EXPECT_CALL(*client_, GetFileInfo(_, _, _)) + .WillOnce(DoAll( + SetArgPointee<2>(fInfo), + Return(LIBCURVE_ERROR::OK))); + + ASSERT_EQ(core_->CheckFileExists("filename", 10), kErrCodeFileNotExist); + + EXPECT_CALL(*client_, GetFileInfo(_, _, _)) + .WillOnce(Return(-LIBCURVE_ERROR::NOTEXIST)); + + ASSERT_EQ(core_->CheckFileExists("filename", 100), kErrCodeFileNotExist); + + EXPECT_CALL(*client_, GetFileInfo(_, _, _)) + .WillOnce(Return(LIBCURVE_ERROR::INTERNAL_ERROR)); + + ASSERT_EQ(core_->CheckFileExists("filename", 100), kErrCodeInternalError); +} + +TEST_F(TestCloneCoreImpl, + TestHandleDeleteCloneInfoSnapDeleteCloneInfoFail) { + CloneInfo info("id1", "user1", CloneTaskType::kClone, + "snapid1", "file1", CloneFileType::kSnapshot, false); + EXPECT_CALL(*metaStore_, DeleteCloneInfo(_)) + .WillOnce(Return(-1)); + snapshotRef_->IncrementSnapshotRef("snapid1"); + ASSERT_EQ(core_->HandleDeleteCloneInfo(info), kErrCodeInternalError); + ASSERT_EQ(snapshotRef_->GetSnapshotRef("snapid1"), 1); +} + +TEST_F(TestCloneCoreImpl, + TestHandleDeleteCloneInfoSnapSuccess) { + CloneInfo info("id1", "user1", CloneTaskType::kClone, + "snapid1", "file1", CloneFileType::kSnapshot, false); + info.SetStatus(CloneStatus::metaInstalled); + EXPECT_CALL(*metaStore_, DeleteCloneInfo(_)) + .WillOnce(Return(0)); + snapshotRef_->IncrementSnapshotRef("snapid1"); + ASSERT_EQ(core_->HandleDeleteCloneInfo(info), kErrCodeSuccess); + ASSERT_EQ(snapshotRef_->GetSnapshotRef("snapid1"), 0); +} + +TEST_F(TestCloneCoreImpl, + TestHandleDeleteCloneInfoFileRefReturnMetainstalledNotTo0) { + CloneInfo info("id1", "user1", CloneTaskType::kClone, + "source1", "file1", CloneFileType::kFile, false); + info.SetStatus(CloneStatus::metaInstalled); + EXPECT_CALL(*metaStore_, DeleteCloneInfo(_)) + .WillOnce(Return(0)); + cloneRef_->IncrementRef("source1"); + cloneRef_->IncrementRef("source1"); + ASSERT_EQ(cloneRef_->GetRef("source1"), 2); + ASSERT_EQ(core_->HandleDeleteCloneInfo(info), kErrCodeSuccess); + ASSERT_EQ(cloneRef_->GetRef("source1"), 1); +} + +TEST_F(TestCloneCoreImpl, + TestHandleDeleteCloneInfoFileSetStatusFail) { + CloneInfo info("id1", "user1", CloneTaskType::kClone, + "source1", "file1", CloneFileType::kFile, false); + info.SetStatus(CloneStatus::metaInstalled); + cloneRef_->IncrementRef("source1"); + ASSERT_EQ(cloneRef_->GetRef("source1"), 1); + EXPECT_CALL(*client_, SetCloneFileStatus(_, _, _)) + .WillOnce(Return(-1)); + ASSERT_EQ(core_->HandleDeleteCloneInfo(info), kErrCodeInternalError); + ASSERT_EQ(cloneRef_->GetRef("source1"), 1); +} + +TEST_F(TestCloneCoreImpl, + TestHandleDeleteCloneInfoFileDeleteCloneInfoFail) { + CloneInfo info("id1", "user1", CloneTaskType::kClone, + "source1", "file1", CloneFileType::kFile, false); + info.SetStatus(CloneStatus::metaInstalled); + EXPECT_CALL(*metaStore_, DeleteCloneInfo(_)) + .WillOnce(Return(-1)); + cloneRef_->IncrementRef("source1"); + ASSERT_EQ(cloneRef_->GetRef("source1"), 1); + EXPECT_CALL(*client_, SetCloneFileStatus(_, _, _)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + ASSERT_EQ(core_->HandleDeleteCloneInfo(info), kErrCodeInternalError); + ASSERT_EQ(cloneRef_->GetRef("source1"), 1); +} + +TEST_F(TestCloneCoreImpl, + TestHandleDeleteCloneInfoFileSuccess) { + CloneInfo info("id1", "user1", CloneTaskType::kClone, + "source1", "file1", CloneFileType::kFile, false); + info.SetStatus(CloneStatus::metaInstalled); + EXPECT_CALL(*metaStore_, DeleteCloneInfo(_)) + .WillOnce(Return(0)); + cloneRef_->IncrementRef("source1"); + ASSERT_EQ(cloneRef_->GetRef("source1"), 1); + EXPECT_CALL(*client_, SetCloneFileStatus(_, _, _)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + ASSERT_EQ(core_->HandleDeleteCloneInfo(info), kErrCodeSuccess); + ASSERT_EQ(cloneRef_->GetRef("source1"), 0); +} + } // namespace snapshotcloneserver } // namespace curve diff --git a/test/snapshotcloneserver/test_clone_service_manager.cpp b/test/snapshotcloneserver/test_clone_service_manager.cpp index b4be909097..2b9bdfdd2b 100644 --- a/test/snapshotcloneserver/test_clone_service_manager.cpp +++ b/test/snapshotcloneserver/test_clone_service_manager.cpp @@ -24,7 +24,7 @@ #include #include "src/snapshotcloneserver/clone/clone_service_manager.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "test/snapshotcloneserver/mock_snapshot_server.h" #include "src/common/concurrent/count_down_event.h" @@ -51,6 +51,8 @@ class TestCloneServiceManager : public ::testing::Test { virtual void SetUp() { cloneCore_ = std::make_shared(); cloneMetric_ = std::make_shared(); + cloneServiceManagerBackend_ = + std::make_shared(); std::shared_ptr cloneTaskMgr_ = std::make_shared(cloneCore_, cloneMetric_); @@ -58,20 +60,31 @@ class TestCloneServiceManager : public ::testing::Test { option_.stage2PoolThreadNum = 3; option_.commonPoolThreadNum = 3; option_.cloneTaskManagerScanIntervalMs = 100; + option_.backEndReferenceRecordScanIntervalMs = 100; + option_.backEndReferenceFuncScanIntervalMs = 1000; manager_ = std::make_shared( - cloneTaskMgr_, cloneCore_); + cloneTaskMgr_, cloneCore_, cloneServiceManagerBackend_); + + EXPECT_CALL(*cloneServiceManagerBackend_, Init(_, _)) + .Times(1); ASSERT_EQ(0, manager_->Init(option_)) << "manager init fail."; + + EXPECT_CALL(*cloneServiceManagerBackend_, Start()) + .Times(1); ASSERT_EQ(0, manager_->Start()) << "manager start fail."; } virtual void TearDown() { + EXPECT_CALL(*cloneServiceManagerBackend_, Stop()) + .Times(1); + manager_->Stop(); + cloneServiceManagerBackend_ = nullptr; cloneCore_ = nullptr; cloneMetric_ = nullptr; - manager_->Stop(); manager_ = nullptr; } @@ -79,6 +92,7 @@ class TestCloneServiceManager : public ::testing::Test { std::shared_ptr cloneCore_; std::shared_ptr manager_; std::shared_ptr cloneMetric_; + std::shared_ptr cloneServiceManagerBackend_; SnapshotCloneServerOptions option_; }; @@ -1064,6 +1078,349 @@ TEST_F(TestCloneServiceManager, TestRecoverCloneTaskDefaultSuccess) { ASSERT_EQ(kErrCodeSuccess, ret); } +TEST_F(TestCloneServiceManager, TestGetCloneRefStatusSuccessNoRef) { + const UUID source = "/file"; + const UUID source2 = "/file2"; + const std::string user = "user1"; + const std::string destination = "file1"; + bool lazyFlag = true; + + // only done record + CloneInfo cloneInfo1("uuid1", user, CloneTaskType::kClone, + source, destination, CloneFileType::kFile, lazyFlag); + cloneInfo1.SetStatus(CloneStatus::done); + CloneInfo cloneInfo2("uuid2", user, CloneTaskType::kClone, + source2, destination, CloneFileType::kFile, lazyFlag); + cloneInfo2.SetStatus(CloneStatus::metaInstalled); + + std::vector list; + list.push_back(cloneInfo1); + list.push_back(cloneInfo2); + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillOnce(DoAll(SetArgPointee<0>(list), + Return(kErrCodeSuccess))); + + std::vector infos; + CloneRefStatus refStatus; + auto ret = manager_->GetCloneRefStatus(source, &refStatus, &infos); + + ASSERT_EQ(kErrCodeSuccess, ret); + ASSERT_EQ(refStatus, CloneRefStatus::kNoRef); + ASSERT_EQ(0, infos.size()); + + // no record found + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillOnce(Return(-1)); + ret = manager_->GetCloneRefStatus(source, &refStatus, &infos); + + ASSERT_EQ(kErrCodeSuccess, ret); + ASSERT_EQ(refStatus, CloneRefStatus::kNoRef); + ASSERT_EQ(0, infos.size()); +} + +TEST_F(TestCloneServiceManager, TestGetCloneRefStatusSuccessHasRef) { + const UUID source = "/file"; + const std::string user = "user1"; + const std::string destination = "file1"; + bool lazyFlag = true; + + CloneInfo cloneInfo1("uuid1", user, CloneTaskType::kClone, + source, destination, CloneFileType::kFile, lazyFlag); + cloneInfo1.SetStatus(CloneStatus::done); + CloneInfo cloneInfo2("uuid2", user, CloneTaskType::kClone, + source, destination, CloneFileType::kFile, lazyFlag); + cloneInfo2.SetStatus(CloneStatus::cloning); + CloneInfo cloneInfo3("uuid3", user, CloneTaskType::kClone, + source, destination, CloneFileType::kFile, lazyFlag); + cloneInfo3.SetStatus(CloneStatus::metaInstalled); + + std::vector list; + list.push_back(cloneInfo1); + list.push_back(cloneInfo2); + list.push_back(cloneInfo3); + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillOnce(DoAll(SetArgPointee<0>(list), + Return(kErrCodeSuccess))); + + std::vector infos; + CloneRefStatus refStatus; + auto ret = manager_->GetCloneRefStatus(source, &refStatus, &infos); + + ASSERT_EQ(kErrCodeSuccess, ret); + ASSERT_EQ(refStatus, CloneRefStatus::kHasRef); + ASSERT_EQ(0, infos.size()); +} + +TEST_F(TestCloneServiceManager, TestGetCloneRefStatusSuccessNeedCheck) { + const UUID source = "/file1"; + const UUID source2 = "/file2"; + const std::string user = "user1"; + const std::string destination = "file1"; + bool lazyFlag = true; + + CloneInfo cloneInfo1("uuid1", user, CloneTaskType::kClone, + source , destination, CloneFileType::kFile, lazyFlag); + cloneInfo1.SetStatus(CloneStatus::done); + CloneInfo cloneInfo2("uuid2", user, CloneTaskType::kClone, + source2, destination, CloneFileType::kFile, lazyFlag); + cloneInfo2.SetStatus(CloneStatus::cloning); + CloneInfo cloneInfo3("uuid3", user, CloneTaskType::kClone, + source, destination, CloneFileType::kFile, lazyFlag); + cloneInfo3.SetStatus(CloneStatus::metaInstalled); + + std::vector list; + list.push_back(cloneInfo1); + list.push_back(cloneInfo2); + list.push_back(cloneInfo3); + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillOnce(DoAll(SetArgPointee<0>(list), + Return(kErrCodeSuccess))); + + std::vector infos; + CloneRefStatus refStatus; + auto ret = manager_->GetCloneRefStatus(source, &refStatus, &infos); + + ASSERT_EQ(kErrCodeSuccess, ret); + ASSERT_EQ(refStatus, CloneRefStatus::kNeedCheck); + ASSERT_EQ(1, infos.size()); + CloneInfo cInfo = infos[0]; + ASSERT_EQ("uuid3", cInfo.GetTaskId()); + ASSERT_EQ(user, cInfo.GetUser()); + ASSERT_EQ(CloneTaskType::kClone, cInfo.GetTaskType()); + ASSERT_EQ(source, cInfo.GetSrc()); + ASSERT_EQ(destination, cInfo.GetDest()); + ASSERT_EQ(CloneFileType::kFile, cInfo.GetFileType()); + ASSERT_EQ(lazyFlag, cInfo.GetIsLazy()); + ASSERT_EQ(CloneStatus::metaInstalled, cInfo.GetStatus()); +} + +class TestCloneServiceManagerBackend : public ::testing::Test { + public: + TestCloneServiceManagerBackend() {} + virtual ~TestCloneServiceManagerBackend() {} + + virtual void SetUp() { + cloneCore_ = std::make_shared(); + cloneServiceManagerBackend_ = + std::make_shared(cloneCore_); + } + + virtual void TearDown() { + cloneServiceManagerBackend_ = nullptr; + cloneCore_ = nullptr; + } + + protected: + std::shared_ptr cloneCore_; + std::shared_ptr cloneServiceManagerBackend_; +}; + +TEST_F(TestCloneServiceManagerBackend, + TestInitStop) { + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + cloneServiceManagerBackend_->Stop(); + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + cloneServiceManagerBackend_->Stop(); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestEmptyCloneInfoList) { + std::vector cloneInfos; + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestNoMetaInstalledClone) { + CloneInfo cloneInfo1("taskId1", "user1", CloneTaskType::kClone, + "source1", "destination1", CloneFileType::kSnapshot, true); + CloneInfo cloneInfo2("taskId2", "user2", CloneTaskType::kRecover, + "source2", "destination2", CloneFileType::kFile, false); + std::vector cloneInfos; + cloneInfos.push_back(cloneInfo1); + cloneInfos.push_back(cloneInfo2); + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestMetaInstalledCloneNotExist) { + CloneInfo cloneInfo1("taskId1", "user1", CloneTaskType::kClone, + "source1", "destination1", CloneFileType::kSnapshot, true); + CloneInfo cloneInfo2("taskId2", "user2", CloneTaskType::kRecover, + "source2", "destination2", 0, 0, 0, CloneFileType::kFile, false, + CloneStep::kRecoverChunk, CloneStatus::metaInstalled); + std::vector cloneInfos; + cloneInfos.push_back(cloneInfo1); + cloneInfos.push_back(cloneInfo2); + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, GetCloneInfo(_, _)) + .WillRepeatedly(Return(-1)); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestMetaInstalledCloneNotMetaInstalled) { + CloneInfo cloneInfo1("taskId1", "user1", CloneTaskType::kClone, + "source1", "destination1", CloneFileType::kSnapshot, true); + CloneInfo cloneInfo2("taskId2", "user2", CloneTaskType::kRecover, + "source2", "destination2", 0, 0, 0, CloneFileType::kFile, false, + CloneStep::kRecoverChunk, CloneStatus::metaInstalled); + std::vector cloneInfos; + cloneInfos.push_back(cloneInfo1); + cloneInfos.push_back(cloneInfo2); + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, GetCloneInfo(_, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(cloneInfo1), + Return(kErrCodeSuccess))); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestMetaInstalledCloneFileExist) { + CloneInfo cloneInfo1("taskId1", "user1", CloneTaskType::kClone, + "source1", "destination1", CloneFileType::kSnapshot, true); + CloneInfo cloneInfo2("taskId2", "user2", CloneTaskType::kRecover, + "source2", "destination2", 0, 0, 0, CloneFileType::kFile, false, + CloneStep::kRecoverChunk, CloneStatus::metaInstalled); + std::vector cloneInfos; + cloneInfos.push_back(cloneInfo1); + cloneInfos.push_back(cloneInfo2); + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, GetCloneInfo(_, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(cloneInfo2), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, CheckFileExists(_, _)) + .WillRepeatedly(Return(kErrCodeFileExist)); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestMetaInstalledCloneDeleteFail) { + CloneInfo cloneInfo1("taskId1", "user1", CloneTaskType::kClone, + "source1", "destination1", CloneFileType::kSnapshot, true); + CloneInfo cloneInfo2("taskId2", "user2", CloneTaskType::kRecover, + "source2", "destination2", 0, 0, 0, CloneFileType::kFile, false, + CloneStep::kRecoverChunk, CloneStatus::metaInstalled); + std::vector cloneInfos; + cloneInfos.push_back(cloneInfo1); + cloneInfos.push_back(cloneInfo2); + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, GetCloneInfo(_, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(cloneInfo2), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, CheckFileExists(_, _)) + .WillRepeatedly(Return(kErrCodeFileNotExist)); + EXPECT_CALL(*cloneCore_, HandleDeleteCloneInfo(_)) + .WillRepeatedly(Return(kErrCodeInternalError)); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + +TEST_F(TestCloneServiceManagerBackend, + TestMetaInstalledCloneDeleteSuccess) { + CloneInfo cloneInfo1("taskId1", "user1", CloneTaskType::kClone, + "source1", "destination1", CloneFileType::kSnapshot, true); + CloneInfo cloneInfo2("taskId2", "user2", CloneTaskType::kRecover, + "source2", "destination2", 0, 0, 0, CloneFileType::kFile, false, + CloneStep::kRecoverChunk, CloneStatus::metaInstalled); + std::vector cloneInfos; + cloneInfos.push_back(cloneInfo1); + cloneInfos.push_back(cloneInfo2); + + uint32_t backEndReferenceRecordScanIntervalMs = 100; + uint32_t backEndReferenceFuncScanIntervalMs = 1000; + uint32_t testCount = 5; + cloneServiceManagerBackend_->Init(backEndReferenceRecordScanIntervalMs, + backEndReferenceFuncScanIntervalMs); + + EXPECT_CALL(*cloneCore_, GetCloneInfoList(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(cloneInfos), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, GetCloneInfo(_, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(cloneInfo2), + Return(kErrCodeSuccess))); + EXPECT_CALL(*cloneCore_, CheckFileExists(_, _)) + .WillRepeatedly(Return(kErrCodeFileNotExist)); + EXPECT_CALL(*cloneCore_, HandleDeleteCloneInfo(_)) + .WillRepeatedly(Return(kErrCodeSuccess)); + cloneServiceManagerBackend_->Start(); + usleep(testCount * backEndReferenceRecordScanIntervalMs * 1000); + cloneServiceManagerBackend_->Stop(); +} + } // namespace snapshotcloneserver } // namespace curve diff --git a/test/snapshotcloneserver/test_snapshot_core.cpp b/test/snapshotcloneserver/test_snapshot_core.cpp index 8e2684cb79..1b4c8d2946 100644 --- a/test/snapshotcloneserver/test_snapshot_core.cpp +++ b/test/snapshotcloneserver/test_snapshot_core.cpp @@ -24,7 +24,7 @@ #include #include "src/snapshotcloneserver/snapshot/snapshot_core.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "src/snapshotcloneserver/snapshot/snapshot_task.h" #include "test/snapshotcloneserver/mock_snapshot_server.h" diff --git a/test/snapshotcloneserver/test_snapshot_service_manager.cpp b/test/snapshotcloneserver/test_snapshot_service_manager.cpp index ac62a2eb69..b1ea2b4859 100644 --- a/test/snapshotcloneserver/test_snapshot_service_manager.cpp +++ b/test/snapshotcloneserver/test_snapshot_service_manager.cpp @@ -24,7 +24,7 @@ #include #include "src/snapshotcloneserver/snapshot/snapshot_service_manager.h" -#include "src/snapshotcloneserver/common/define.h" +#include "src/common/snapshotclone/snapshotclone_define.h" #include "test/snapshotcloneserver/mock_snapshot_server.h" #include "src/common/concurrent/count_down_event.h" @@ -447,6 +447,7 @@ TEST_F(TestSnapshotServiceManager, TestDeleteSnapshotByCancelSuccess) { EXPECT_CALL(*core_, HandleCreateSnapshotTask(_)) .WillOnce(Invoke([&cond1] ( std::shared_ptr task) { + LOG(INFO) << "in HandleCreateSnapshotTask"; while (1) { if (task->IsCanceled()) { break; @@ -1065,6 +1066,7 @@ TEST_F(TestSnapshotServiceManager, EXPECT_CALL(*core_, HandleCreateSnapshotTask(_)) .WillOnce(Invoke([&cond1, &cond2] ( std::shared_ptr task) { + LOG(INFO) << "in mock HandleCreateSnapshotTask"; while (1) { if (task->IsCanceled()) { cond1.Signal(); diff --git a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp index b3217172f4..2dbf55db45 100644 --- a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp +++ b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp @@ -413,6 +413,9 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, int ret = metaStore_->AddCloneInfo(cloneInfo); ASSERT_EQ(0, ret); + EXPECT_CALL(*kvStorageClient_, Get(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*kvStorageClient_, Put(_, _)) .WillOnce(Return(EtcdErrCode::EtcdOK)); @@ -448,6 +451,9 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, int ret = metaStore_->AddCloneInfo(cloneInfo); ASSERT_EQ(0, ret); + EXPECT_CALL(*kvStorageClient_, Get(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*kvStorageClient_, Put(_, _)) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); @@ -456,7 +462,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, } TEST_F(TestSnapshotCloneMetaStoreEtcd, - TestUpdateCloneInfoNotExistAndGetSuccess) { + TestUpdateCloneInfoNotExist) { CloneInfo cloneInfo("uuid1", "user1", CloneTaskType::kClone, "src1", "dst1", 1, 2, 3, @@ -464,17 +470,11 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, CloneStep::kCompleteCloneFile, CloneStatus::cloning); - EXPECT_CALL(*kvStorageClient_, Put(_, _)) - .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*kvStorageClient_, Get(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); int ret = metaStore_->UpdateCloneInfo(cloneInfo); - ASSERT_EQ(0, ret); - - CloneInfo outInfo; - ret = metaStore_->GetCloneInfo("uuid1", &outInfo); - ASSERT_EQ(0, ret); - - ASSERT_TRUE(JudgeCloneInfoEqual(cloneInfo, outInfo)); + ASSERT_EQ(-1, ret); } TEST_F(TestSnapshotCloneMetaStoreEtcd, diff --git a/test/snapshotcloneserver/test_snapshotclone_service.cpp b/test/snapshotcloneserver/test_snapshotclone_service.cpp index df578a6e34..4595a950b2 100644 --- a/test/snapshotcloneserver/test_snapshotclone_service.cpp +++ b/test/snapshotcloneserver/test_snapshotclone_service.cpp @@ -1747,6 +1747,261 @@ TEST_F(TestSnapshotCloneServiceImpl, TestFlattenFail) { LOG(ERROR) << cntl.response_attachment(); } +TEST_F(TestSnapshotCloneServiceImpl, TestGetCloneRefStatusNeedCheckRefSuccess) { + std::string user = "user1"; + std::string source = "/file"; + + std::vector infoVec; + CloneInfo cinfo1, cinfo2, cinfo3, cinfo4; + cinfo1.SetTaskId("1"); + cinfo1.SetUser("1"); + cinfo1.SetDest("1"); + cinfo1.SetDestId(1); + + cinfo2.SetTaskId("2"); + cinfo2.SetUser("2"); + cinfo2.SetDest("2"); + cinfo2.SetDestId(2); + + cinfo3.SetTaskId("3"); + cinfo3.SetUser("3"); + cinfo3.SetDest("3"); + cinfo3.SetDestId(3); + + cinfo4.SetTaskId("4"); + cinfo4.SetUser("4"); + cinfo4.SetDest("4"); + cinfo4.SetDestId(4); + + infoVec.push_back(cinfo1); + infoVec.push_back(cinfo2); + infoVec.push_back(cinfo3); + infoVec.push_back(cinfo4); + + CloneRefStatus refStatus = CloneRefStatus::kNeedCheck; + EXPECT_CALL(*cloneManager_, GetCloneRefStatus(source, _, _)) + .WillOnce(DoAll( + SetArgPointee<1>(refStatus), + SetArgPointee<2>(infoVec), + Return(kErrCodeSuccess))); + + brpc::Channel channel; + brpc::ChannelOptions option; + option.protocol = "http"; + std::string url = std::string("http://127.0.0.1:") + + std::to_string(listenAddr_.port) + + "/" + kServiceName + "?" + + kActionStr + "=" + kGetCloneRefStatusAction + "&" + + kVersionStr + "=1&" + + kUserStr + "=" + user + "&" + + kSourceStr + "=" + source; + + if (channel.Init(url.c_str(), "", &option) != 0) { + FAIL() << "Fail to init channel" + << std::endl; + } + + brpc::Controller cntl; + cntl.http_request().uri() = url.c_str(); + + channel.CallMethod(NULL, &cntl, NULL, NULL, NULL); + if (cntl.Failed()) { + LOG(ERROR) << cntl.ErrorText(); + } + LOG(ERROR) << cntl.response_attachment(); + + std::stringstream ss; + ss << cntl.response_attachment(); + std::string data = ss.str(); + Json::Reader jsonReader; + Json::Value jsonObj; + if (!jsonReader.parse(data, jsonObj)) { + FAIL() << "parse json fail, data = " << data; + } + ASSERT_STREQ("0", jsonObj["Code"].asCString()); + ASSERT_EQ(4, jsonObj["TotalCount"].asInt()); + ASSERT_EQ(2, jsonObj["RefStatus"].asInt()); + ASSERT_EQ(4, jsonObj["CloneFileInfo"].size()); + ASSERT_STREQ("1", jsonObj["CloneFileInfo"][0]["User"].asCString()); + ASSERT_STREQ("1", jsonObj["CloneFileInfo"][0]["File"].asCString()); + ASSERT_EQ(1, jsonObj["CloneFileInfo"][0]["Inode"].asUInt64()); + ASSERT_STREQ("2", jsonObj["CloneFileInfo"][1]["User"].asCString()); + ASSERT_STREQ("3", jsonObj["CloneFileInfo"][2]["User"].asCString()); + ASSERT_STREQ("4", jsonObj["CloneFileInfo"][3]["User"].asCString()); +} + +TEST_F(TestSnapshotCloneServiceImpl, TestGetCloneRefStatusNoRefSuccess) { + std::string user = "user1"; + std::string source = "/file"; + + CloneRefStatus refStatus = CloneRefStatus::kNoRef; + EXPECT_CALL(*cloneManager_, GetCloneRefStatus(source, _, _)) + .WillOnce(DoAll( + SetArgPointee<1>(refStatus), + Return(kErrCodeSuccess))); + + brpc::Channel channel; + brpc::ChannelOptions option; + option.protocol = "http"; + std::string url = std::string("http://127.0.0.1:") + + std::to_string(listenAddr_.port) + + "/" + kServiceName + "?" + + kActionStr + "=" + kGetCloneRefStatusAction + "&" + + kVersionStr + "=1&" + + kUserStr + "=" + user + "&" + + kSourceStr + "=" + source; + + if (channel.Init(url.c_str(), "", &option) != 0) { + FAIL() << "Fail to init channel" + << std::endl; + } + + brpc::Controller cntl; + cntl.http_request().uri() = url.c_str(); + + channel.CallMethod(NULL, &cntl, NULL, NULL, NULL); + if (cntl.Failed()) { + LOG(ERROR) << cntl.ErrorText(); + } + LOG(ERROR) << cntl.response_attachment(); + + std::stringstream ss; + ss << cntl.response_attachment(); + std::string data = ss.str(); + Json::Reader jsonReader; + Json::Value jsonObj; + if (!jsonReader.parse(data, jsonObj)) { + FAIL() << "parse json fail, data = " << data; + } + ASSERT_STREQ("0", jsonObj["Code"].asCString()); + ASSERT_EQ(0, jsonObj["TotalCount"].asInt()); + ASSERT_EQ(0, jsonObj["RefStatus"].asInt()); +} + +TEST_F(TestSnapshotCloneServiceImpl, TestGetCloneRefStatusHasRefSuccess) { + std::string user = "user1"; + std::string source = "/file"; + + std::vector infoVec; + CloneInfo cinfo1, cinfo2, cinfo3, cinfo4; + cinfo1.SetTaskId("1"); + cinfo2.SetTaskId("2"); + cinfo3.SetTaskId("3"); + cinfo4.SetTaskId("4"); + infoVec.push_back(cinfo1); + infoVec.push_back(cinfo2); + infoVec.push_back(cinfo3); + infoVec.push_back(cinfo4); + + CloneRefStatus refStatus = CloneRefStatus::kHasRef; + EXPECT_CALL(*cloneManager_, GetCloneRefStatus(source, _, _)) + .WillOnce(DoAll( + SetArgPointee<1>(refStatus), + Return(kErrCodeSuccess))); + + brpc::Channel channel; + brpc::ChannelOptions option; + option.protocol = "http"; + std::string url = std::string("http://127.0.0.1:") + + std::to_string(listenAddr_.port) + + "/" + kServiceName + "?" + + kActionStr + "=" + kGetCloneRefStatusAction + "&" + + kVersionStr + "=1&" + + kUserStr + "=" + user + "&" + + kSourceStr + "=" + source; + + if (channel.Init(url.c_str(), "", &option) != 0) { + FAIL() << "Fail to init channel" + << std::endl; + } + + brpc::Controller cntl; + cntl.http_request().uri() = url.c_str(); + + channel.CallMethod(NULL, &cntl, NULL, NULL, NULL); + if (cntl.Failed()) { + LOG(ERROR) << cntl.ErrorText(); + } + LOG(ERROR) << cntl.response_attachment(); + + std::stringstream ss; + ss << cntl.response_attachment(); + std::string data = ss.str(); + Json::Reader jsonReader; + Json::Value jsonObj; + if (!jsonReader.parse(data, jsonObj)) { + FAIL() << "parse json fail, data = " << data; + } + ASSERT_STREQ("0", jsonObj["Code"].asCString()); + ASSERT_EQ(0, jsonObj["TotalCount"].asInt()); + ASSERT_EQ(1, jsonObj["RefStatus"].asInt()); +} + +TEST_F(TestSnapshotCloneServiceImpl, TestGetCloneRefStatusMissingParam) { + std::string user = "user1"; + std::string source = "/file"; + + brpc::Channel channel; + brpc::ChannelOptions option; + option.protocol = "http"; + std::string url = std::string("http://127.0.0.1:") + + std::to_string(listenAddr_.port) + + "/" + kServiceName + "?" + + kActionStr + "=" + kGetCloneRefStatusAction + "&" + + kVersionStr + "=1&"; + + if (channel.Init(url.c_str(), "", &option) != 0) { + FAIL() << "Fail to init channel" + << std::endl; + } + + brpc::Controller cntl; + cntl.http_request().uri() = url.c_str(); + + channel.CallMethod(NULL, &cntl, NULL, NULL, NULL); + if (cntl.Failed()) { + LOG(ERROR) << cntl.ErrorText(); + } + LOG(ERROR) << cntl.response_attachment(); + ASSERT_EQ(brpc::HTTP_STATUS_BAD_REQUEST, + cntl.http_response().status_code()); +} + +TEST_F(TestSnapshotCloneServiceImpl, TestGetCloneRefStatusFail) { + std::string user = "user1"; + std::string source = "/file"; + + EXPECT_CALL(*cloneManager_, GetCloneRefStatus(source, _, _)) + .WillOnce(Return(kErrCodeInternalError)); + + brpc::Channel channel; + brpc::ChannelOptions option; + option.protocol = "http"; + std::string url = std::string("http://127.0.0.1:") + + std::to_string(listenAddr_.port) + + "/" + kServiceName + "?" + + kActionStr + "=" + kGetCloneRefStatusAction + "&" + + kVersionStr + "=1&" + + kUserStr + "=" + user + "&" + + kSourceStr + "=" + source; + + if (channel.Init(url.c_str(), "", &option) != 0) { + FAIL() << "Fail to init channel" + << std::endl; + } + + brpc::Controller cntl; + cntl.http_request().uri() = url.c_str(); + + channel.CallMethod(NULL, &cntl, NULL, NULL, NULL); + if (cntl.Failed()) { + LOG(ERROR) << cntl.ErrorText(); + } + LOG(ERROR) << cntl.response_attachment(); + + ASSERT_EQ(brpc::HTTP_STATUS_INTERNAL_SERVER_ERROR, + cntl.http_response().status_code()); +} } // namespace snapshotcloneserver } // namespace curve