curvebs: support 512 aligned IO

Signed-off-by: Hanqing Wu <wuhanqing@corp.netease.com>

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings

conf/chunkserver.conf.example

@@ -13,6 +13,11 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
 
 #

conf/mds.conf

@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config

deploy/local/chunkserver/conf/chunkserver.conf.0

@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

deploy/local/chunkserver/conf/chunkserver.conf.1

@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

deploy/local/chunkserver/conf/chunkserver.conf.2

@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

include/BUILD

@@ -18,7 +18,7 @@ load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
 
 cc_library(
     name = "include-common",
-    srcs = ["curve_compiler_specific.h"],
+    hdrs = ["curve_compiler_specific.h"],
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
 )

include/chunkserver/chunkserver_common.h

@@ -123,8 +123,11 @@ inline std::string ToGroupIdString(const LogicPoolID &logicPoolId,
 }
 #define ToGroupIdStr   ToGroupIdString
 
-// TODO(wudmeiao): 是否需要考虑可配置
-const uint32_t kOpRequestAlignSize = 4096;
+// Meta page is header of chunkfile, and is used to store meta data of
+// chunkfile.
+// Currently, we need to ensure the atomicity of the meta page update, so set
+// its size to 4k.
+constexpr size_t kChunkfileMetaPageSize = 4096;
 
 }  // namespace chunkserver
 }  // namespace curve

include/client/libcurve.h

@@ -54,6 +54,7 @@ typedef struct FileStatInfo {
     int             fileStatus;
     uint64_t        stripeUnit;
     uint64_t        stripeCount;
+    uint32_t        blocksize;
 } FileStatInfo_t;
 
 // 存储用户信息
@@ -438,7 +439,7 @@ class CurveClient {
      * @param fd 文件fd
      * @return 返回错误码
      */
-    virtual int64_t StatFile(int fd);
+    virtual int64_t StatFile(int fd, FileStatInfo* fileStat);
 
     /**
      * 异步读

nebd/proto/client.proto

@@ -98,6 +98,7 @@ message FileInfo {
    required uint64 size = 1;
    required uint64 objSize = 2;
    required uint64 objNums = 3;
+   optional uint32 blocksize = 4;
 }
 
 message GetInfoRequest {

nebd/src/part1/libnebd.cpp

@@ -118,6 +118,10 @@ int64_t nebd_lib_filesize(int fd) {
     return GetFileSize4Nebd(fd);
 }
 
+int64_t nebd_lib_blocksize(int fd) {
+    return GetBlockSize4Nebd(fd);
+}
+
 int nebd_lib_resize(int fd, int64_t size) {
     return Extend4Nebd(fd, size);
 }

nebd/src/part1/libnebd.h

@@ -159,6 +159,8 @@ int nebd_lib_sync(int fd);
  */
 int64_t nebd_lib_filesize(int fd);
 
+int64_t nebd_lib_blocksize(int fd);
+
 /**
  *  @brief resize文件
  *  @param fd：文件的fd

nebd/src/part1/libnebd_file.cpp

@@ -51,6 +51,10 @@ int64_t GetFileSize4Nebd(int fd) {
     return nebd::client::nebdClient.GetFileSize(fd);
 }
 
+int64_t GetBlockSize4Nebd(int fd) {
+    return nebd::client::nebdClient.GetBlockSize(fd);
+}
+
 int Discard4Nebd(int fd, NebdClientAioContext* aioctx) {
     return nebd::client::nebdClient.Discard(fd, aioctx);
 }

nebd/src/part1/libnebd_file.h

@@ -62,6 +62,9 @@ int Extend4Nebd(int fd, int64_t newsize);
  *  @return 成功返回文件size，失败返回错误码
  */
 int64_t GetFileSize4Nebd(int fd);
+
+int64_t GetBlockSize4Nebd(int fd);
+
 /**
  *  @brief discard文件，异步函数
  *  @param fd：文件的fd

nebd/src/part1/nebd_client.cpp

@@ -320,6 +320,44 @@ int64_t NebdClient::GetFileSize(int fd) {
     return ret;
 }
 
+int64_t NebdClient::GetBlockSize(int fd) {
+    auto task = [&](brpc::Controller* cntl, brpc::Channel* channel,
+                    bool* rpcFailed) -> int64_t {
+        nebd::client::NebdFileService_Stub stub(channel);
+        nebd::client::GetInfoRequest request;
+        nebd::client::GetInfoResponse response;
+
+        request.set_fd(fd);
+        stub.GetInfo(cntl, &request, &response, nullptr);
+
+        *rpcFailed = cntl->Failed();
+        if (*rpcFailed) {
+            LOG(WARNING) << "GetBlockSize failed, error = " << cntl->ErrorText()
+                         << ", log id = " << cntl->log_id();
+            return -1;
+        } else {
+            if (response.retcode() != nebd::client::RetCode::kOK) {
+                LOG(ERROR) << "GetBlockSize failed, "
+                           << "retcode = " << response.retcode()
+                           << ",  retmsg = " << response.retmsg()
+                           << ", fd = " << fd
+                           << ", log id = " << cntl->log_id();
+                return -1;
+            } else {
+                return response.info().has_blocksize()
+                           ? response.info().blocksize()
+                           : kDefaultBlockSize;
+            }
+        }
+    };
+
+    int64_t ret = ExecuteSyncRpc(task);
+    if (ret < 0) {
+        LOG(ERROR) << "GetBlockSize failed, fd = " << fd;
+    }
+    return ret;
+}
+
 int NebdClient::Discard(int fd, NebdClientAioContext* aioctx) {
     auto task = [this, fd, aioctx]() {
         nebd::client::NebdFileService_Stub stub(&channel_);

nebd/src/part1/nebd_client.h

@@ -100,6 +100,8 @@ class NebdClient {
      */
     int64_t GetFileSize(int fd);
 
+    int64_t GetBlockSize(int fd);
+
     /**
      *  @brief discard文件，异步函数
      *  @param fd：文件的fd

nebd/src/part1/nebd_common.h

@@ -71,4 +71,6 @@ struct HeartbeatOption {
     int64_t rpcTimeoutMs;
 };
 
+constexpr uint32_t kDefaultBlockSize = 4096;
+
 #endif  // NEBD_SRC_PART1_NEBD_COMMON_H_

nebd/src/part2/define.h

@@ -105,6 +105,8 @@ struct NebdFileInfo {
     uint64_t obj_size;
     // object数量
     uint64_t num_objs;
+    // block size
+    uint32_t block_size;
 };
 
 using ExtendAttribute = std::map<std::string, std::string>;

nebd/src/part2/file_service.cpp

@@ -298,6 +298,7 @@ void NebdFileServiceImpl::GetInfo(
         info->set_size(fileInfo.size);
         info->set_objsize(fileInfo.obj_size);
         info->set_objnums(fileInfo.num_objs);
+        info->set_blocksize(fileInfo.block_size);
         response->set_retcode(RetCode::kOK);
         response->set_allocated_info(info);
     }

nebd/src/part2/request_executor_curve.cpp

@@ -189,12 +189,14 @@ int CurveRequestExecutor::GetInfo(
         return -1;
     }
 
-    int64_t size = client_->StatFile(curveFd);
-    if (size < 0) {
+    FileStatInfo statInfo;
+    int64_t rc = client_->StatFile(curveFd, &statInfo);
+    if (rc < 0) {
         return -1;
     }
 
-    fileInfo->size = size;
+    fileInfo->size = statInfo.length;
+    fileInfo->block_size = statInfo.blocksize;
     return 0;
 }
 

nebd/test/part1/fake_file_service.cpp

@@ -157,6 +157,7 @@ void FakeNebdFileService::GetInfo(::google::protobuf::RpcController* controller,
     info->set_size(fileSize_);
     info->set_objsize(fileSize_);
     info->set_objnums(1);
+    info->set_blocksize(blockSize_);
     response->set_retcode(RetCode::kOK);
     response->set_retmsg("GetInfo OK");
     response->set_allocated_info(info);

nebd/test/part1/fake_file_service.h

@@ -27,13 +27,15 @@
 #include <brpc/controller.h>
 #include <string>
 #include "nebd/proto/client.pb.h"
+#include "nebd/src/part1/nebd_common.h"
 
 namespace nebd {
 namespace client {
 
 class FakeNebdFileService: public NebdFileService {
  public:
-    FakeNebdFileService() {}
+    explicit FakeNebdFileService(uint32_t blockSize = kDefaultBlockSize)
+        : fileSize_(0), blockSize_(blockSize) {}
 
     virtual ~FakeNebdFileService() {}
 
@@ -84,6 +86,7 @@ class FakeNebdFileService: public NebdFileService {
 
  private:
     int64_t fileSize_;
+    uint32_t blockSize_;
 };
 }  // namespace client
 }  // namespace nebd

nebd/test/part1/nebd_client_unittest.cpp

@@ -269,6 +269,7 @@ TEST_F(NebdFileClientTest, NoNebdServerTest) {
     }
     ASSERT_EQ(-1, Extend4Nebd(1, kFileSize));
     ASSERT_EQ(-1, GetFileSize4Nebd(1));
+    ASSERT_EQ(-1, GetBlockSize4Nebd(1));
     ASSERT_EQ(-1, GetInfo4Nebd(1));
     ASSERT_EQ(-1, InvalidCache4Nebd(1));
     ASSERT_EQ(-1, Close4Nebd(1));
@@ -292,6 +293,7 @@ TEST_F(NebdFileClientTest, CommonTest) {
     ASSERT_EQ(0, Extend4Nebd(fd, kFileSize));
     ASSERT_EQ(kFileSize, GetFileSize4Nebd(fd));
     ASSERT_EQ(kFileSize, GetInfo4Nebd(fd));
+    ASSERT_EQ(kDefaultBlockSize, GetBlockSize4Nebd(fd));
     ASSERT_EQ(0, InvalidCache4Nebd(fd));
 
     char buffer[kBufSize];
@@ -443,6 +445,17 @@ TEST_F(NebdFileClientTest, ResponseFailTest) {
         ASSERT_EQ(-1, GetFileSize4Nebd(1));
     }
 
+    {
+        GetInfoResponse response;
+        response.set_retcode(RetCode::kNoOK);
+        EXPECT_CALL(mockService, GetInfo(_, _, _, _))
+            .Times(1)
+            .WillOnce(DoAll(
+                SetArgPointee<2>(response),
+                Invoke(MockClientFunc<GetInfoRequest, GetInfoResponse>)));
+        ASSERT_EQ(-1, GetBlockSize4Nebd(1));
+    }
+
     {
         GetInfoResponse response;
         response.set_retcode(RetCode::kNoOK);

nebd/test/part1/nebd_lib_unittest.cpp

@@ -41,6 +41,7 @@ const char* kFileName = "nebd-lib-test-filename";
 const char* kNebdServerTestAddress = "./nebd-lib-unittest.sock";
 const char* kNebdClientConf = "./nebd/test/part1/nebd-lib-test.conf";
 const int64_t kFileSize = 10LL * 1024 * 1024 * 1024;
+const uint32_t kBlockSize = 4096;
 const int64_t kBufSize = 1024;
 
 namespace nebd {
@@ -89,7 +90,7 @@ class NebdLibTest : public ::testing::Test {
     }
 
     brpc::Server server;
-    FakeNebdFileService fakeService;
+    FakeNebdFileService fakeService{kBlockSize};
     MockNebdFileService mockService;
 };
 
@@ -106,6 +107,7 @@ TEST_F(NebdLibTest, CommonTest) {
     ASSERT_EQ(0, nebd_lib_resize(fd, kFileSize));
     ASSERT_EQ(kFileSize, nebd_lib_filesize(fd));
     ASSERT_EQ(kFileSize, nebd_lib_getinfo(fd));
+    ASSERT_EQ(kBlockSize, nebd_lib_blocksize(fd));
     ASSERT_EQ(0, nebd_lib_invalidcache(fd));
 
     ASSERT_EQ(-1, nebd_lib_pread(fd, 0, 0, 0));

nebd/test/part2/mock_curve_client.h

@@ -41,7 +41,7 @@ class MockCurveClient : public ::curve::client::CurveClient {
                  int(const std::string&, const ::curve::client::OpenFlags&));
     MOCK_METHOD1(Close, int(int));
     MOCK_METHOD2(Extend, int(const std::string&, int64_t));
-    MOCK_METHOD1(StatFile, int64_t(int));
+    MOCK_METHOD2(StatFile, int64_t(int, FileStatInfo*));
     MOCK_METHOD3(AioRead,
                  int(int, CurveAioContext*, curve::client::UserDataType));
     MOCK_METHOD3(AioWrite,