curvefs/metaserver: fix copysets can't reload concurrently and suppor…

…t dynamic vlog

curvefs/conf/metaserver.conf

@@ -71,7 +71,7 @@ copyset.data_uri=local://./0/copysets  # __CURVEADM_TEMPLATE__ local://${prefix}
 # if value set to 1, means all copysets are loaded one by one it may cause a long start-up time
 # if value bigger than 1, means at most |load_concurrency| copysets are loaded parallelly
 # but larger value may cause higher cpu/memory/disk usgae
-copyset.load_concurrency=1
+copyset.load_concurrency=5
 
 # if the difference between the applied_index of the current replica and the
 # committed_index on the leader is less than |finishLoadMargin|, it's

curvefs/src/client/BUILD

@@ -67,6 +67,7 @@ cc_library(
         "//src/common:curve_s3_adapter",
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
+        "//curvefs/src/common:dynamic_vlog",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/meta:type_traits",

curvefs/src/client/curve_fuse_op.cpp

@@ -38,6 +38,7 @@
 #include "curvefs/src/client/rpcclient/base_client.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "curvefs/src/common/metric_utils.h"
+#include "curvefs/src/common/dynamic_vlog.h"
 
 using ::curve::common::Configuration;
 using ::curvefs::client::CURVEFS_ERROR;
@@ -52,27 +53,12 @@ using ::curvefs::client::metric::ClientOpMetric;
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::client::metric::InflightGuard;
 
+using ::curvefs::common::FLAGS_vlog_level;
+
 static FuseClient *g_ClientInstance = nullptr;
 static FuseClientOption *g_fuseClientOption = nullptr;
 static ClientOpMetric* g_clientOpMetric = nullptr;
 
-DECLARE_int32(v);
-
-/**
- * use vlog_level to set vlog level on the fly
- * When vlog_level is set, CheckVLogLevel is called to check the validity of the
- * value. Dynamically modify the vlog level by setting FLAG_v in CheckVLogLevel.
- *
- * You can modify the vlog level to 0 using:
- * curl -s http://127.0.0.1:9000/flags/vlog_level?setvalue=0
- */
-DEFINE_int32(vlog_level, 0, "set vlog level");
-static bool CheckVLogLevel(const char*, int32_t value) {
-    FLAGS_v = value;
-    return true;
-}
-DEFINE_validator(vlog_level, CheckVLogLevel);
-
 namespace {
 
 void EnableSplice(struct fuse_conn_info* conn) {

curvefs/src/common/BUILD

@@ -18,10 +18,16 @@ load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
 
 cc_library(
     name = "curvefs_common",
-    srcs = glob(["*.cpp"]),
+    srcs = glob(
+        ["*.cpp"],
+        exclude = ["dynamic_vlog.cpp"],
+    ),
     hdrs = glob(
         ["*.h"],
-        exclude = ["metric_utils.h"],
+        exclude = [
+            "metric_utils.h",
+            "dynamic_vlog.h",
+        ],
     ),
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
@@ -35,12 +41,29 @@ cc_library(
 cc_library(
     name = "metric_utils",
     hdrs = [
-        "metric_utils.h"
+        "metric_utils.h",
     ],
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
     deps = [
-        "//external:bvar",
         "//external:butil",
-    ]
+        "//external:bvar",
+    ],
+)
+
+cc_library(
+    name = "dynamic_vlog",
+    srcs = [
+        "dynamic_vlog.cpp",
+    ],
+    hdrs = [
+        "dynamic_vlog.h",
+    ],
+    copts = CURVE_DEFAULT_COPTS,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//external:brpc",
+        "//external:gflags",
+        "//external:glog",
+    ],
 )

curvefs/src/common/dynamic_vlog.cpp

@@ -0,0 +1,45 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Thursday May 19 10:03:22 CST 2022
+ * Author: wuhanqing
+ */
+
+#include "curvefs/src/common/dynamic_vlog.h"
+
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+
+DECLARE_int32(v);
+
+namespace curvefs {
+namespace common {
+
+namespace {
+bool CheckVLogLevel(const char* /*name*/, int32_t value) {
+    FLAGS_v = value;
+    LOG(INFO) << "current verbose logging level is `" << FLAGS_v << "`";
+    return true;
+}
+}  // namespace
+
+DEFINE_int32(vlog_level, 0, "set vlog level");
+DEFINE_validator(vlog_level, CheckVLogLevel);
+
+}  // namespace common
+}  // namespace curvefs

curvefs/src/common/dynamic_vlog.h

@@ -0,0 +1,44 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Thursday May 19 10:17:45 CST 2022
+ * Author: wuhanqing
+ */
+
+#ifndef CURVEFS_SRC_COMMON_DYNAMIC_VLOG_H_
+#define CURVEFS_SRC_COMMON_DYNAMIC_VLOG_H_
+
+#include <gflags/gflags.h>
+
+namespace curvefs {
+namespace common {
+
+/**
+ * use vlog_level to set vlog level on the fly
+ * When vlog_level is set, CheckVLogLevel is called to check the validity of the
+ * value. Dynamically modify the vlog level by setting FLAG_v in CheckVLogLevel.
+ *
+ * You can modify the vlog level to 0 using:
+ * curl -s http://127.0.0.1:9000/flags/vlog_level?setvalue=0
+ */
+DECLARE_int32(vlog_level);
+
+}  // namespace common
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_COMMON_DYNAMIC_VLOG_H_

curvefs/src/metaserver/BUILD

@@ -54,6 +54,7 @@ cc_library(
         "//external:braft",
         "//src/common:curve_common",
         "//src/fs:lfs",
+        "//curvefs/src/common:dynamic_vlog",
         "@rocksdb//:rocksdb_lib",
         "@com_google_absl//absl/cleanup",
         "@com_google_absl//absl/container:btree",

curvefs/src/metaserver/copyset/copyset_node_manager.cpp

@@ -209,27 +209,37 @@ bool CopysetNodeManager::CreateCopysetNode(PoolId poolId, CopysetId copysetId,
     braft::GroupId groupId = ToGroupId(poolId, copysetId);
     std::shared_ptr<CopysetNode> copysetNode;
 
-    WriteLockGuard lock(lock_);
-    if (copysets_.count(groupId) != 0) {
-        LOG(WARNING) << "Copyset node already exists: "
-                     << ToGroupIdString(poolId, copysetId);
-        return false;
-    }
+    {
+        WriteLockGuard lock(lock_);
+        if (copysets_.count(groupId) != 0) {
+            LOG(WARNING) << "Copyset node already exists: "
+                         << ToGroupIdString(poolId, copysetId);
+            return false;
+        }
 
-    copysetNode = std::make_shared<CopysetNode>(poolId, copysetId, conf, this);
-    if (!copysetNode->Init(options_)) {
-        LOG(ERROR) << "Copyset " << ToGroupIdString(poolId, copysetId)
-                   << "init failed";
-        return false;
+        copysetNode =
+            std::make_shared<CopysetNode>(poolId, copysetId, conf, this);
+        if (!copysetNode->Init(options_)) {
+            LOG(ERROR) << "Copyset " << ToGroupIdString(poolId, copysetId)
+                       << "init failed";
+            return false;
+        }
+
+        copysets_.emplace(groupId, copysetNode);
     }
 
+    // node start maybe time-consuming
     if (!copysetNode->Start()) {
+        // checkLoadFinish equals to false when reloading copysets after a
+        // restart, in this case we should not automaticaly remove copyset's
+        // data
+        const bool removeData = checkLoadFinish;
+        DeleteCopysetNodeInternal(poolId, copysetId, removeData);
         LOG(ERROR) << "Copyset " << ToGroupIdString(poolId, copysetId)
                    << " start failed";
         return false;
     }
 
-    copysets_.emplace(groupId, std::move(copysetNode));
     LOG(INFO) << "Create copyset success "
               << ToGroupIdString(poolId, copysetId);
     return true;
@@ -244,7 +254,6 @@ void CopysetNodeManager::GetAllCopysets(
     }
 }
 
-// TODO(wuhanqing): disgingush internal server and external server
 void CopysetNodeManager::AddService(brpc::Server* server,
                                     const butil::EndPoint& listenAddr) {
     braft::add_service(server, listenAddr);

curvefs/src/metaserver/dentry_manager.cpp

@@ -36,7 +36,7 @@ DentryManager::DentryManager(std::shared_ptr<DentryStorage> dentryStorage,
 
 void DentryManager::Log4Dentry(const std::string& request,
                                const Dentry& dentry) {
-    VLOG(1) << "Receive " <<  request << " request, dentry = ("
+    VLOG(9) << "Receive " <<  request << " request, dentry = ("
             << dentry.ShortDebugString() << ")";
 }
 
@@ -50,7 +50,7 @@ void DentryManager::Log4Code(const std::string& request, MetaStatusCode rc) {
             << ", retCode = " << MetaStatusCode_Name(rc);
 
     if (succ) {
-        VLOG(1) << message.str();
+        VLOG(6) << message.str();
     } else {
         LOG(ERROR) << message.str();
     }

curvefs/src/metaserver/dentry_storage.cpp

@@ -241,7 +241,7 @@ MetaStatusCode DentryStorage::List(const Dentry& dentry,
             count++;
             auto iter = temp->rbegin();
             dentrys->push_back(*iter);
-            VLOG(1) << "ListDentry, dentry = ("
+            VLOG(9) << "ListDentry, dentry = ("
                     << iter->ShortDebugString() << ")";
         }
         temp->clear();