Skip to content

Commit

Permalink
support discard
Browse files Browse the repository at this point in the history
  • Loading branch information
wu-hanqing committed Dec 20, 2020
1 parent 7f12926 commit 4617fd2
Show file tree
Hide file tree
Showing 78 changed files with 2,910 additions and 309 deletions.
18 changes: 9 additions & 9 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")

git_repository(
name = "com_github_baidu_braft",
remote = "https://github.com/baidu/braft",
remote = "https://gitee.com/baidu/braft",
commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e",
)

Expand All @@ -36,7 +36,7 @@ http_archive(
name = "com_google_protobuf",
sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30",
strip_prefix = "protobuf-3.5.0",
urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"],
urls = ["https://curve-build.nos-eastchina1.126.net/protobuf-3.5.0.zip"],
)

bind(
Expand All @@ -48,7 +48,7 @@ bind(
new_git_repository(
name = "com_google_googletest",
build_file = "bazel/gmock.BUILD",
remote = "https://github.com/google/googletest",
remote = "https://gitee.com/mirrors/googletest",
tag = "release-1.8.0",
)

Expand All @@ -61,7 +61,7 @@ bind(
# brpc内BUILD文件在依赖glog时, 直接指定的依赖是"@com_github_google_glog//:glog"
git_repository(
name = "com_github_google_glog",
remote = "https://github.com/google/glog",
remote = "https://gitee.com/vNext/glog",
commit = "4cc89c9e2b452db579397887c37f302fb28f6ca1",
patch_args = ["-p1"],
patches = ["//:thirdparties/glog/glog.patch"],
Expand All @@ -77,7 +77,7 @@ http_archive(
name = "com_github_gflags_gflags",
strip_prefix = "gflags-2.2.2",
urls = [
"https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz",
"https://curve-build.nos-eastchina1.126.net/gflags-2.2.2.tar.gz",
"https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
],
)
Expand All @@ -91,7 +91,7 @@ new_http_archive(
name = "com_github_google_leveldb",
build_file = "bazel/leveldb.BUILD",
strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
url = "https://curve-build.nos-eastchina1.126.net/leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
)

bind(
Expand All @@ -101,7 +101,7 @@ bind(

git_repository(
name = "com_github_apache_brpc",
remote = "https://github.com/apache/incubator-brpc",
remote = "https://gitee.com/baidu/BRPC",
commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
patches = ["//:thirdparties/brpc/brpc.patch"],
patch_args = ["-p1"],
Expand Down Expand Up @@ -131,7 +131,7 @@ bind(
new_git_repository(
name = "jsoncpp",
build_file = "bazel/jsoncpp.BUILD",
remote = "https://github.com/open-source-parsers/jsoncpp.git",
remote = "https://gitee.com/mirrors/jsoncpp",
tag = "1.8.4",
)

Expand All @@ -149,7 +149,7 @@ new_local_repository(
new_http_archive(
name = "aws",
urls = [
"https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
"https://curve-build.nos-eastchina1.126.net/aws-sdk-cpp-1.7.340.tar.gz",
"https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
],
sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
Expand Down
10 changes: 10 additions & 0 deletions conf/client.conf
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,13 @@ global.metricDummyServerStartPort=9000

# 是否关闭健康检查: true/关闭 false/不关闭
global.turnOffHealthCheck=true

# 是否开启discard
discard.enableDiscard=true

# discard粒度
discard.discardGranularity=4096

# discard清理任务延迟时间(毫秒)
discard.discardTaskDelayMs=60000

2 changes: 2 additions & 0 deletions conf/mds.conf
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ mds.segment.alloc.periodic.persistInterMs=10000
# 出错情况下的重试间隔,单位ms
mds.segment.alloc.retryInterMs=1000

mds.segment.discard.scanIntevalMs=5000


# leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s)
# 该值和etcd集群election timeout相关.
Expand Down
9 changes: 9 additions & 0 deletions include/client/libcurve.h
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ const char* ErrorNum2ErrorName(LIBCURVE_ERROR err);
typedef enum LIBCURVE_OP {
LIBCURVE_OP_READ,
LIBCURVE_OP_WRITE,
LIBCURVE_OP_DISCARD,
LIBCURVE_OP_MAX,
} LIBCURVE_OP;

Expand Down Expand Up @@ -481,6 +482,14 @@ class CurveClient {
virtual int AioWrite(int fd, CurveAioContext* aioctx,
UserDataType dataType);

/**
* @brief Async Discard
* @param fd file descriptor
* @param aioctx async request context
* @return return error code, 0 means success
*/
virtual int AioDiscard(int fd, CurveAioContext* aioctx);

/**
* 测试使用,设置fileclient
* @param client 需要设置的fileclient
Expand Down
18 changes: 18 additions & 0 deletions proto/nameserver2.proto
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ message PageFileSegment {
repeated PageFileChunkInfo chunks = 5;
}

message DiscardSegmentInfo {
required FileInfo fileInfo = 1;
required PageFileSegment pageFileSegment = 2;
}

message CreateFileRequest {
required string fileName = 1;
required FileType fileType = 3;
Expand Down Expand Up @@ -211,6 +216,18 @@ message GetOrAllocateSegmentResponse {
optional PageFileSegment pageFileSegment = 2;
}

message DeAllocateSegmentRequest {
required string fileName = 1;
required string owner = 2;
required uint64 offset = 3;
optional string signature = 4;
required uint64 date = 5;
}

message DeAllocateSegmentResponse {
required StatusCode statusCode = 1;
}

message RenameFileRequest {
required string oldFileName = 1;
required string newFileName = 2;
Expand Down Expand Up @@ -496,6 +513,7 @@ service CurveFSService {
rpc GetFileInfo(GetFileInfoRequest) returns (GetFileInfoResponse);
rpc GetOrAllocateSegment(GetOrAllocateSegmentRequest)
returns (GetOrAllocateSegmentResponse);
rpc DeAllocateSegment(DeAllocateSegmentRequest) returns (DeAllocateSegmentResponse);
rpc RenameFile(RenameFileRequest) returns (RenameFileResponse);
rpc ExtendFile(ExtendFileRequest) returns (ExtendFileResponse);
rpc ChangeOwner(ChangeOwnerRequest) returns (ChangeOwnerResponse);
Expand Down
8 changes: 8 additions & 0 deletions src/client/client_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,17 @@ using CopysetID = uint32_t;
using LogicPoolID = uint32_t;
using ChunkServerID = uint32_t;
using ChunkIndex = uint32_t;
using SegmentIndex = uint32_t;

using EndPoint = butil::EndPoint;
using Status = butil::Status;

using IOManagerID = uint64_t;

constexpr uint64_t KiB = 1024;
constexpr uint64_t MiB = 1024 * KiB;
constexpr uint64_t GiB = 1024 * MiB;

// 操作类型
enum class OpType {
READ = 0,
Expand All @@ -56,6 +61,7 @@ enum class OpType {
CREATE_CLONE,
RECOVER_CHUNK,
GET_CHUNK_INFO,
DISCARD,
UNKNOWN
};

Expand Down Expand Up @@ -214,6 +220,8 @@ inline const char* OpTypeToString(OpType optype) {
return "RecoverChunk";
case OpType::GET_CHUNK_INFO:
return "GetChunkInfo";
case OpType::DISCARD:
return "Discard";
case OpType::UNKNOWN:
default:
return "Unknown";
Expand Down
22 changes: 22 additions & 0 deletions src/client/client_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ namespace curve {
namespace client {
int ClientConfig::Init(const char* configpath) {
conf_.SetConfigPath(configpath);

LOG(INFO) << "Init config from " << configpath;

if (!conf_.LoadConfig()) {
LOG(ERROR) << "Load config failed, config path = " << configpath;
return -1;
Expand Down Expand Up @@ -218,6 +221,25 @@ int ClientConfig::Init(const char* configpath) {
<< "config no global.turnOffHealthCheck info, using default value "
<< fileServiceOption_.commonOpt.turnOffHealthCheck;

ret = conf_.GetBoolValue(
"discard.enableDiscard",
&fileServiceOption_.ioOpt.discardOption.enableDiscard);
LOG_IF(ERROR, ret == false)
<< "config no discard.enableDiscard info";
RETURN_IF_FALSE(ret);

ret = conf_.GetUInt32Value("discard.discardGranularity",
&fileServiceOption_.ioOpt.metaCacheOpt.discardGranularity);
LOG_IF(ERROR, ret == false)
<< "config no discard.discardGranularity info";
RETURN_IF_FALSE(ret);

ret = conf_.GetUInt32Value("discard.discardTaskDelayMs",
&fileServiceOption_.ioOpt.discardOption.discardTaskDelayMs);
LOG_IF(ERROR, ret == false)
<< "config no discard.discardTaskDelayMs info";
RETURN_IF_FALSE(ret);

return 0;
}

Expand Down
3 changes: 3 additions & 0 deletions src/client/client_metric.h
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ struct MDSClientMetric {
InterfaceMetric getServerList;
// GetOrAllocateSegment接口统计信息
InterfaceMetric getOrAllocateSegment;

InterfaceMetric deAllocateSegment;
// RenameFile接口统计信息
InterfaceMetric renameFile;
// Extend接口统计信息
Expand Down Expand Up @@ -185,6 +187,7 @@ struct MDSClientMetric {
refreshSession(prefix, "refreshSession"),
getServerList(prefix, "getServerList"),
getOrAllocateSegment(prefix, "getOrAllocateSegment"),
deAllocateSegment(prefix, "deAllocateSegment"),
renameFile(prefix, "renameFile"),
extendFile(prefix, "extendFile"),
deleteFile(prefix, "deleteFile"),
Expand Down
9 changes: 9 additions & 0 deletions src/client/config_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ struct MetaCacheOption {
uint32_t metacacheRPCRetryIntervalUS = 500;
uint32_t metacacheGetLeaderRPCTimeOutMS = 1000;
uint32_t metacacheGetLeaderBackupRequestMS = 100;
uint32_t discardGranularity = 4096;
std::string metacacheGetLeaderBackupRequestLbName = "rr";
ChunkServerUnstableOption chunkserverUnstableOption;
};
Expand All @@ -213,6 +214,12 @@ struct TaskThreadOption {
uint32_t isolationTaskThreadPoolSize = 1;
};

// for discard
struct DiscardOption {
bool enableDiscard = false;
uint32_t discardTaskDelayMs = 1000 * 60 * 3; // 3 min
};

/**
* IOOption存储了当前io 操作所需要的所有配置信息
*/
Expand All @@ -222,6 +229,7 @@ struct IOOption {
MetaCacheOption metaCacheOpt;
TaskThreadOption taskThreadOpt;
RequestScheduleOption reqSchdulerOpt;
DiscardOption discardOption;
};

/**
Expand Down Expand Up @@ -255,6 +263,7 @@ struct FileServiceOption {
LeaseOption leaseOpt;
CommonConfigOpt commonOpt;
MetaServerOption metaServerOpt;
DiscardOption discardOption;
};

} // namespace client
Expand Down
64 changes: 64 additions & 0 deletions src/client/discard_task.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Copyright (c) 2020 NetEase Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* Project: curve
* File Created: Thu Dec 17 11:05:38 CST 2020
* Author: wuhanqing
*/

#include "src/client/discard_task.h"

namespace curve {
namespace client {

std::atomic<uint64_t> DiscardTask::taskId_(1);

bool DiscardTask::OnTriggeringTask(timespec*) {
const FInfo* fileInfo = metaCache_->GetFileInfo();
uint64_t offset =
static_cast<uint64_t>(segmentIndex_) * fileInfo->segmentsize;
FileSegmentInfo* fileSegment =
metaCache_->GetFileSegmentInfo(segmentIndex_);

FileSegmentWriteLockGuard lk(fileSegment);

if (!fileSegment->IsAllDiscard()) {
LOG(WARNING) << "DiscardTask find bitmap was cleared, cancel task, "
"filename = "
<< fileInfo->fullPathName << ", offset = " << offset
<< ", taskid = " << id_;
return false;
}

LIBCURVE_ERROR errCode = mdsClient_->DeAllocateSegment(fileInfo, offset);
if (errCode == LIBCURVE_ERROR::OK) {
fileSegment->ClearBitmap();
metaCache_->CleanChunksInSegment(segmentIndex_);
LOG(INFO) << "DiscardTask success, filename = "
<< fileInfo->fullPathName << ", offset = " << offset
<< ", taskid = " << id_;
} else {
LOG(ERROR) << "DiscardTask failed, mds return error = " << errCode
<< ", filename = " << fileInfo->fullPathName
<< ", offset = " << offset << ", taskid = " << id_;
}

return false;
}

} // namespace client
} // namespace curve
Loading

0 comments on commit 4617fd2

Please sign in to comment.