From 3caf951d5714374c0059cc51a33de81b187f5424 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Thu, 15 Mar 2018 12:25:56 -0400 Subject: [PATCH 01/54] Edit Group API --- plugins/device/api/req.go | 4 +++ plugins/device/api/resp.go | 2 ++ plugins/device/init.go | 13 +++++++- plugins/device/rest.go | 62 +++++++++++++++++++++++++++++++++++ plugins/device/transaction.go | 31 ++++++++++++++++++ 5 files changed, 111 insertions(+), 1 deletion(-) diff --git a/plugins/device/api/req.go b/plugins/device/api/req.go index 29fabc803..be5d94538 100644 --- a/plugins/device/api/req.go +++ b/plugins/device/api/req.go @@ -15,3 +15,7 @@ const ( type AddDeviceReq struct { Devices []string `json:"devices"` } + +type EditGroupReq struct { + Group string `json:"group"` +} diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index ecdc932ea..266c3f8c4 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -12,3 +12,5 @@ type Info struct { // AddDeviceResp is the success response sent to a AddDeviceReq request type AddDeviceResp api.Peer + +type EditGroupResp api.Peer diff --git a/plugins/device/init.go b/plugins/device/init.go index 2315f9c4c..9bdf4313e 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -32,7 +32,17 @@ func (p *Plugin) RestRoutes() route.Routes { Version: 1, RequestType: utils.GetTypeString((*deviceapi.AddDeviceReq)(nil)), ResponseType: utils.GetTypeString((*deviceapi.AddDeviceResp)(nil)), - HandlerFunc: deviceAddHandler}, + HandlerFunc: deviceAddHandler, + }, + route.Route{ + Name: "EditGroup", + Method: "POST", + Pattern: "/peers/{peerid}/group/{group_id}", + Version: 1, + RequestType: utils.GetTypeString((*deviceapi.EditGroupReq)(nil)), + ResponseType: utils.GetTypeString((*deviceapi.EditGroupResp)(nil)), + HandlerFunc: groupEditHandler, + }, } } @@ -40,4 +50,5 @@ func (p *Plugin) RestRoutes() route.Routes { // Glusterd Transaction framework func (p *Plugin) RegisterStepFuncs() { transaction.RegisterStepFunc(txnPrepareDevice, "prepare-device") + transaction.RegisterStepFunc(txnEditGroup, "edit-group") } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 51f919ebc..655fe5d87 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -74,3 +74,65 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { } restutils.SendHTTPResponse(ctx, w, http.StatusOK, peerInfo) } + +func groupEditHandler(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + logger := gdctx.GetReqLogger(ctx) + + req := new(deviceapi.EditGroupReq) + if err := restutils.UnmarshalRequest(r, req); err != nil { + logger.WithError(err).Error("Failed to Unmarshal request") + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Unable to marshal request", api.ErrCodeDefault) + return + } + + peerID := mux.Vars(r)["peerid"] + if peerID == "" { + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "peerid not present in request", api.ErrCodeDefault) + return + } + groupID := mux.Vars(r)["group_id"] + txn := transaction.NewTxn(ctx) + defer txn.Cleanup() + lock, unlock, err := transaction.CreateLockSteps(peerID) + txn.Steps = []*transaction.Step{ + lock, + { + DoFunc: "edit-group", + Nodes: []uuid.UUID{gdctx.MyUUID}, + }, + unlock, + } + err = txn.Ctx.Set("peerid", peerID) + if err != nil { + logger.WithError(err).Error("Failed to set data for transaction") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err.Error(), api.ErrCodeDefault) + return + } + err = txn.Ctx.Set("groupid", groupID) + if err != nil { + logger.WithError(err).Error("Failed to set data for transaction") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err.Error(), api.ErrCodeDefault) + return + } + err = txn.Ctx.Set("req", req) + if err != nil { + logger.WithError(err).Error("Failed to set data for transaction") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err.Error(), api.ErrCodeDefault) + return + } + err = txn.Do() + if err != nil { + logger.WithError(err).Error("Transaction to edit group failed") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Transaction to edit group failed", api.ErrCodeDefault) + return + } + peerInfo, err := peer.GetPeer(peerID) + if err != nil { + logger.WithError(err).Error("Failed to get peer from store") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Failed to get peer from store", api.ErrCodeDefault) + return + } + restutils.SendHTTPResponse(ctx, w, http.StatusOK, peerInfo) +} diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index 2967418b0..f68afe416 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -4,6 +4,7 @@ import ( "os/exec" "strings" + "github.com/gluster/glusterd2/glusterd2/peer" "github.com/gluster/glusterd2/glusterd2/transaction" deviceapi "github.com/gluster/glusterd2/plugins/device/api" @@ -51,3 +52,33 @@ func txnPrepareDevice(c transaction.TxnCtx) error { } return nil } + +func txnEditGroup(c transaction.TxnCtx) error { + var peerID string + var groupID string + var req deviceapi.EditGroupReq + if err := c.Get("peerid", peerID); err != nil { + c.Logger().WithError(err).Error("Failed transaction, cannot find peer-id") + return err + } + if err := c.Get("groupid", groupID); err != nil { + c.Logger().WithError(err).Error("Failed transaction, cannot find group-id") + return err + } + if err := c.Get("req", req); err != nil { + c.Logger().WithError(err).Error("Failed transaction, cannot find group details") + return err + } + peerInfo, err := peer.GetPeer(peerID) + if err != nil { + c.Logger().WithError(err).WithField("peerid", peerID).Error("Peer ID not found in store") + return err + } + peerInfo.MetaData["_group"] = req.Group + err = peer.AddOrUpdatePeer(peerInfo) + if err != nil { + c.Logger().WithError(err).WithField("peerid", peerID).Error("Failed to update peer Info") + return err + } + return nil +} From 76324c9fe16fcbb73adf5ddfb834c23aae44dee7 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Thu, 15 Mar 2018 17:13:58 -0400 Subject: [PATCH 02/54] Updating code --- plugins/device/api/req.go | 3 ++- plugins/device/api/resp.go | 3 ++- plugins/device/init.go | 12 ++++++------ plugins/device/rest.go | 19 ++++++------------- plugins/device/transaction.go | 16 ++++++---------- 5 files changed, 22 insertions(+), 31 deletions(-) diff --git a/plugins/device/api/req.go b/plugins/device/api/req.go index be5d94538..1e2deabc8 100644 --- a/plugins/device/api/req.go +++ b/plugins/device/api/req.go @@ -16,6 +16,7 @@ type AddDeviceReq struct { Devices []string `json:"devices"` } -type EditGroupReq struct { +// PeerEditGroupReq structure +type PeerEditGroupReq struct { Group string `json:"group"` } diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index 266c3f8c4..67a5e71ca 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -13,4 +13,5 @@ type Info struct { // AddDeviceResp is the success response sent to a AddDeviceReq request type AddDeviceResp api.Peer -type EditGroupResp api.Peer +// PeerEditGroupResp is the success response sent to a EditGroup request +type PeerEditGroupResp api.Peer diff --git a/plugins/device/init.go b/plugins/device/init.go index 9bdf4313e..1ef83ca35 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -35,13 +35,13 @@ func (p *Plugin) RestRoutes() route.Routes { HandlerFunc: deviceAddHandler, }, route.Route{ - Name: "EditGroup", + Name: "PeerEditGroup", Method: "POST", - Pattern: "/peers/{peerid}/group/{group_id}", + Pattern: "/peers/{peerid}/group", Version: 1, - RequestType: utils.GetTypeString((*deviceapi.EditGroupReq)(nil)), - ResponseType: utils.GetTypeString((*deviceapi.EditGroupResp)(nil)), - HandlerFunc: groupEditHandler, + RequestType: utils.GetTypeString((*deviceapi.PeerEditGroupReq)(nil)), + ResponseType: utils.GetTypeString((*deviceapi.PeerEditGroupResp)(nil)), + HandlerFunc: peerEditGroupHandler, }, } } @@ -50,5 +50,5 @@ func (p *Plugin) RestRoutes() route.Routes { // Glusterd Transaction framework func (p *Plugin) RegisterStepFuncs() { transaction.RegisterStepFunc(txnPrepareDevice, "prepare-device") - transaction.RegisterStepFunc(txnEditGroup, "edit-group") + transaction.RegisterStepFunc(txnPeerEditGroup, "peer-edit-group") } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 655fe5d87..11af71910 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -75,12 +75,12 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPResponse(ctx, w, http.StatusOK, peerInfo) } -func groupEditHandler(w http.ResponseWriter, r *http.Request) { +func peerEditGroupHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) - req := new(deviceapi.EditGroupReq) + req := new(deviceapi.PeerEditGroupReq) if err := restutils.UnmarshalRequest(r, req); err != nil { logger.WithError(err).Error("Failed to Unmarshal request") restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Unable to marshal request", api.ErrCodeDefault) @@ -92,33 +92,26 @@ func groupEditHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "peerid not present in request", api.ErrCodeDefault) return } - groupID := mux.Vars(r)["group_id"] txn := transaction.NewTxn(ctx) defer txn.Cleanup() lock, unlock, err := transaction.CreateLockSteps(peerID) txn.Steps = []*transaction.Step{ lock, { - DoFunc: "edit-group", + DoFunc: "peer-edit-group", Nodes: []uuid.UUID{gdctx.MyUUID}, }, unlock, } err = txn.Ctx.Set("peerid", peerID) if err != nil { - logger.WithError(err).Error("Failed to set data for transaction") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err.Error(), api.ErrCodeDefault) - return - } - err = txn.Ctx.Set("groupid", groupID) - if err != nil { - logger.WithError(err).Error("Failed to set data for transaction") + logger.WithError(err).WithField("PeerID", peerID).Error("Failed to set data for transaction") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err.Error(), api.ErrCodeDefault) return } err = txn.Ctx.Set("req", req) if err != nil { - logger.WithError(err).Error("Failed to set data for transaction") + logger.WithError(err).WithField("req", req).Error("Failed to set data for transaction") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err.Error(), api.ErrCodeDefault) return } @@ -130,7 +123,7 @@ func groupEditHandler(w http.ResponseWriter, r *http.Request) { } peerInfo, err := peer.GetPeer(peerID) if err != nil { - logger.WithError(err).Error("Failed to get peer from store") + logger.WithError(err).WithField("PeerID", peerID).Error("Failed to get peer from store") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Failed to get peer from store", api.ErrCodeDefault) return } diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index f68afe416..54cb48fb4 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -53,20 +53,16 @@ func txnPrepareDevice(c transaction.TxnCtx) error { return nil } -func txnEditGroup(c transaction.TxnCtx) error { +func txnPeerEditGroup(c transaction.TxnCtx) error { + var peerID string - var groupID string - var req deviceapi.EditGroupReq if err := c.Get("peerid", peerID); err != nil { - c.Logger().WithError(err).Error("Failed transaction, cannot find peer-id") - return err - } - if err := c.Get("groupid", groupID); err != nil { - c.Logger().WithError(err).Error("Failed transaction, cannot find group-id") + c.Logger().WithError(err).WithField("PeerID", peerID).Error("Failed transaction, cannot find peer-id") return err } + var req deviceapi.PeerEditGroupReq if err := c.Get("req", req); err != nil { - c.Logger().WithError(err).Error("Failed transaction, cannot find group details") + c.Logger().WithError(err).WithField("req", req).Error("Failed transaction, cannot find req") return err } peerInfo, err := peer.GetPeer(peerID) @@ -77,7 +73,7 @@ func txnEditGroup(c transaction.TxnCtx) error { peerInfo.MetaData["_group"] = req.Group err = peer.AddOrUpdatePeer(peerInfo) if err != nil { - c.Logger().WithError(err).WithField("peerid", peerID).Error("Failed to update peer Info") + c.Logger().WithError(err).WithField("GroupID", req.Group).WithField("peerid", peerID).Error("Failed to update peer Info") return err } return nil From 9ab82ca69158e234a4d0528452ccb349e672c604 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Sun, 1 Apr 2018 06:59:10 -0400 Subject: [PATCH 03/54] Checking device if it already exists before adding in store --- plugins/device/rest.go | 11 +++++++++-- plugins/device/store-utils.go | 33 +++++++++++++++++++++++++++++++++ plugins/device/transaction.go | 6 +++--- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/plugins/device/rest.go b/plugins/device/rest.go index f1417a69f..692583b82 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -36,6 +36,13 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPError(ctx, w, http.StatusNotFound, "Peer-id not found in store") return } + + devices, err := CheckIfDeviceExist(req.Devices, peerInfo.MetaData["_devices"]) + if err != nil { + logger.WithError(err).WithField("device", req.Devices).Error("Device already exist") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Devices already exist") + return + } txn := transaction.NewTxn(ctx) defer txn.Cleanup() lock, unlock, err := transaction.CreateLockSteps(peerID) @@ -58,9 +65,9 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - err = txn.Ctx.Set("req", &req) + err = txn.Ctx.Set("devices", &devices) if err != nil { - logger.WithError(err).WithField("key", "req").Error("Failed to set key in transaction context") + logger.WithError(err).WithField("key", "devices").Error("Failed to set key in transaction context") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } diff --git a/plugins/device/store-utils.go b/plugins/device/store-utils.go index 02609e16e..b80f516f8 100644 --- a/plugins/device/store-utils.go +++ b/plugins/device/store-utils.go @@ -2,6 +2,7 @@ package device import ( "encoding/json" + "errors" peer "github.com/gluster/glusterd2/glusterd2/peer" deviceapi "github.com/gluster/glusterd2/plugins/device/api" @@ -23,6 +24,38 @@ func GetDevices(peerID string) ([]deviceapi.Info, error) { return nil, nil } +//CheckIfDeviceExist returns error if all devices already exist or returns list of devices to be added +func CheckIfDeviceExist(reqDevices []string, metadataDevices string) ([]string, error) { + + if metadataDevices == "" { + return reqDevices, nil + } + + var devices []deviceapi.Info + err := json.Unmarshal([]byte(metadataDevices), &devices) + if err != nil { + return nil, err + } + var tempDevice []string + var flag bool + for _, key := range reqDevices { + flag = true + for _, reqKey := range devices { + if key == reqKey.Name { + flag = false + break + } + } + if flag { + tempDevice = append(tempDevice, key) + } + } + if len(tempDevice) == 0 { + return nil, errors.New("Devices already added") + } + return tempDevice, nil +} + // AddDevices adds device to specific peer func AddDevices(devices []deviceapi.Info, peerID string) error { deviceDetails, err := GetDevices(peerID) diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index 8e0e828c6..43f004ca1 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -12,18 +12,18 @@ import ( func txnPrepareDevice(c transaction.TxnCtx) error { var peerID string - var req deviceapi.AddDeviceReq + var devices []string var deviceList []deviceapi.Info if err := c.Get("peerid", &peerID); err != nil { c.Logger().WithError(err).WithField("key", "peerid").Error("Failed to get key from transaction context") return err } - if err := c.Get("req", &req); err != nil { + if err := c.Get("devices", &devices); err != nil { c.Logger().WithError(err).WithField("key", "req").Error("Failed to get key from transaction context") return err } - for _, name := range req.Devices { + for _, name := range devices { tempDevice := deviceapi.Info{ Name: name, } From 7ea61200ca4ed068b273c28ff5e8cb47971d4aa3 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Mon, 2 Apr 2018 08:15:11 -0400 Subject: [PATCH 04/54] Handling device setup failures --- glusterd2/commands/peers/editpeer.go | 5 +++ pkg/api/peer_req_resp.go | 1 + plugins/device/api/req.go | 5 --- plugins/device/api/resp.go | 3 -- plugins/device/cmdexec/device.go | 49 +++++++++++++++++++++++ plugins/device/init.go | 9 ----- plugins/device/rest.go | 60 ---------------------------- plugins/device/transaction.go | 19 ++------- 8 files changed, 59 insertions(+), 92 deletions(-) create mode 100644 plugins/device/cmdexec/device.go diff --git a/glusterd2/commands/peers/editpeer.go b/glusterd2/commands/peers/editpeer.go index 83ff4b8e4..bce79cc3f 100644 --- a/glusterd2/commands/peers/editpeer.go +++ b/glusterd2/commands/peers/editpeer.go @@ -101,6 +101,7 @@ func txnPeerEdit(c transaction.TxnCtx) error { c.Logger().WithError(err).WithField("peerid", peerID).Error("Peer ID not found in store") return err } + for k, v := range req.MetaData { if peerInfo.MetaData != nil { peerInfo.MetaData[k] = v @@ -109,6 +110,10 @@ func txnPeerEdit(c transaction.TxnCtx) error { peerInfo.MetaData[k] = v } } + + if req.Zone != "" { + peerInfo.MetaData["_zone"] = req.Zone + } err = peer.AddOrUpdatePeer(peerInfo) if err != nil { c.Logger().WithError(err).WithField("peerid", peerID).Error("Failed to update peer Info") diff --git a/pkg/api/peer_req_resp.go b/pkg/api/peer_req_resp.go index 4a04cbcbc..2fabff3a0 100644 --- a/pkg/api/peer_req_resp.go +++ b/pkg/api/peer_req_resp.go @@ -21,6 +21,7 @@ type PeerAddReq struct { // PeerEditReq represents an incoming request to edit metadata of peer type PeerEditReq struct { + Zone string `json:"zone"` MetaData map[string]string `json:"metadata"` } diff --git a/plugins/device/api/req.go b/plugins/device/api/req.go index 655a1027d..29fabc803 100644 --- a/plugins/device/api/req.go +++ b/plugins/device/api/req.go @@ -15,8 +15,3 @@ const ( type AddDeviceReq struct { Devices []string `json:"devices"` } - -// PeerEditZoneReq structure -type PeerEditZoneReq struct { - Zone string `json:"zone"` -} diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index 2bf61983f..ecdc932ea 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -12,6 +12,3 @@ type Info struct { // AddDeviceResp is the success response sent to a AddDeviceReq request type AddDeviceResp api.Peer - -// PeerEditZoneResp is the success response sent to a EditGroup request -type PeerEditZoneResp api.Peer diff --git a/plugins/device/cmdexec/device.go b/plugins/device/cmdexec/device.go new file mode 100644 index 000000000..b07899d6b --- /dev/null +++ b/plugins/device/cmdexec/device.go @@ -0,0 +1,49 @@ +package cmdexec + +import ( + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" +) + +func createVgName(device string) string { + vgName := strings.Replace("vg"+device, "/", "-", -1) + return vgName +} + +func DeviceSetup(device string) error { + + var err error + pvcreateCmd := exec.Command("pvcreate", "--metadatasize=128M", "--dataalignment=256K", device) + if err := pvcreateCmd.Run(); err != nil { + log.WithError(err).WithField("device", device).Error("pvcreate failed for device") + return err + } + vgName := createVgName(device) + vgcreateCmd := exec.Command("vgcreate", vgName, device) + if err = vgcreateCmd.Run(); err != nil { + log.WithError(err).WithField("device", device).Error("vgcreate failed for device") + return err + } + + defer func() { + if err != nil { + DeleteDevice(device) + } + }() + return nil + +} + +func DeleteDevice(device string) { + vgName := createVgName(device) + vgremoveCmd := exec.Command("vgremove", vgName) + if err := vgremoveCmd.Run(); err != nil { + log.WithError(err).WithField("device", device).Error("vgremove failed for device") + } + pvremoveCmd := exec.Command("pvremove", device) + if err := pvremoveCmd.Run(); err != nil { + log.WithError(err).WithField("device", device).Error("pvremove failed for device") + } +} diff --git a/plugins/device/init.go b/plugins/device/init.go index 6d32e609c..2189b8052 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -34,15 +34,6 @@ func (p *Plugin) RestRoutes() route.Routes { ResponseType: utils.GetTypeString((*deviceapi.AddDeviceResp)(nil)), HandlerFunc: deviceAddHandler, }, - route.Route{ - Name: "PeerEditZone", - Method: "POST", - Pattern: "/peers/{peerid}/zone", - Version: 1, - RequestType: utils.GetTypeString((*deviceapi.PeerEditZoneReq)(nil)), - ResponseType: utils.GetTypeString((*deviceapi.PeerEditZoneResp)(nil)), - HandlerFunc: peerEditZoneHandler, - }, } } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 692583b82..052a28e07 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -7,7 +7,6 @@ import ( "github.com/gluster/glusterd2/glusterd2/peer" restutils "github.com/gluster/glusterd2/glusterd2/servers/rest/utils" "github.com/gluster/glusterd2/glusterd2/transaction" - "github.com/gluster/glusterd2/pkg/api" deviceapi "github.com/gluster/glusterd2/plugins/device/api" "github.com/gorilla/mux" @@ -85,62 +84,3 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { } restutils.SendHTTPResponse(ctx, w, http.StatusOK, peerInfo) } - -func peerEditZoneHandler(w http.ResponseWriter, r *http.Request) { - - ctx := r.Context() - logger := gdctx.GetReqLogger(ctx) - - peerID := mux.Vars(r)["peerid"] - if uuid.Parse(peerID) == nil { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Invalid peer id passed in request url") - return - } - - req := new(deviceapi.PeerEditZoneReq) - if err := restutils.UnmarshalRequest(r, req); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) - return - } - - txn := transaction.NewTxn(ctx) - defer txn.Cleanup() - lock, unlock, err := transaction.CreateLockSteps(peerID) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Steps = []*transaction.Step{ - lock, - { - DoFunc: "peer-edit", - Nodes: []uuid.UUID{gdctx.MyUUID}, - }, - unlock, - } - - err = txn.Ctx.Set("peerid", peerID) - if err != nil { - logger.WithError(err).WithField("key", "peerid").WithField("value", peerID).Error("Failed to set key in transaction context") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - - var editPeerReq api.PeerEditReq - editPeerReq.MetaData = make(map[string]string) - editPeerReq.MetaData["_zone"] = req.Zone - - err = txn.Ctx.Set("req", editPeerReq) - if err != nil { - logger.WithError(err).WithField("key", "req").Error("Failed to set key in transaction context") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - err = txn.Do() - if err != nil { - logger.WithError(err).Error("Transaction to edit zone failed") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Transaction to edit zone failed") - return - } - restutils.SendHTTPResponse(ctx, w, http.StatusOK, nil) -} diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index 43f004ca1..a092ab3e9 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -1,13 +1,9 @@ package device import ( - "os/exec" - "strings" - "github.com/gluster/glusterd2/glusterd2/transaction" deviceapi "github.com/gluster/glusterd2/plugins/device/api" - - log "github.com/sirupsen/logrus" + "github.com/gluster/glusterd2/plugins/device/cmdexec" ) func txnPrepareDevice(c transaction.TxnCtx) error { @@ -30,15 +26,8 @@ func txnPrepareDevice(c transaction.TxnCtx) error { deviceList = append(deviceList, tempDevice) } for index, element := range deviceList { - pvcreateCmd := exec.Command("pvcreate", "--metadatasize=128M", "--dataalignment=256K", element.Name) - if err := pvcreateCmd.Run(); err != nil { - c.Logger().WithError(err).WithField("device", element.Name).Error("pvcreate failed for device") - deviceList[index].State = deviceapi.DeviceFailed - continue - } - vgcreateCmd := exec.Command("vgcreate", strings.Replace("vg"+element.Name, "/", "-", -1), element.Name) - if err := vgcreateCmd.Run(); err != nil { - c.Logger().WithError(err).WithField("device", element.Name).Error("vgcreate failed for device") + err := cmdexec.DeviceSetup(element.Name) + if err != nil { deviceList[index].State = deviceapi.DeviceFailed continue } @@ -46,7 +35,7 @@ func txnPrepareDevice(c transaction.TxnCtx) error { } err := AddDevices(deviceList, peerID) if err != nil { - log.WithError(err).Error("Couldn't add deviceinfo to store") + c.Logger().WithError(err).Error("Couldn't add deviceinfo to store") return err } return nil From 1f94dd3fe8f3373409806bc7d9c3efeb727678e4 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Tue, 3 Apr 2018 07:44:46 -0400 Subject: [PATCH 05/54] Handling loosing metadata when glusterd restarts --- glusterd2/peer/self.go | 24 ++++++++++++++---------- plugins/device/transaction.go | 6 ++++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/glusterd2/peer/self.go b/glusterd2/peer/self.go index 38bf85864..f805ab5eb 100644 --- a/glusterd2/peer/self.go +++ b/glusterd2/peer/self.go @@ -39,25 +39,29 @@ func normalizeAddrs() ([]string, error) { // AddSelfDetails results in the peer adding its own details into etcd func AddSelfDetails() error { - var err error + var err error p := &Peer{ ID: gdctx.MyUUID, Name: gdctx.HostName, PeerAddresses: []string{config.GetString("peeraddress")}, } - - if p.MetaData == nil { - p.MetaData = make(map[string]string) - } - if p.MetaData["_zone"] == "" { - p.MetaData["_zone"] = p.ID.String() - } - p.ClientAddresses, err = normalizeAddrs() if err != nil { return err } + peerInfo, err := GetPeer(string(gdctx.MyUUID)) + if err != nil { - return AddOrUpdatePeer(p) + if p.MetaData == nil { + p.MetaData = make(map[string]string) + } + if p.MetaData["_zone"] == "" { + p.MetaData["_zone"] = p.ID.String() + } + + return AddOrUpdatePeer(p) + } + peerInfo.MetaData = p.MetaData + return nil } diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index a092ab3e9..27af168c0 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -8,17 +8,19 @@ import ( func txnPrepareDevice(c transaction.TxnCtx) error { var peerID string - var devices []string - var deviceList []deviceapi.Info if err := c.Get("peerid", &peerID); err != nil { c.Logger().WithError(err).WithField("key", "peerid").Error("Failed to get key from transaction context") return err } + + var devices []string if err := c.Get("devices", &devices); err != nil { c.Logger().WithError(err).WithField("key", "req").Error("Failed to get key from transaction context") return err } + + var deviceList []deviceapi.Info for _, name := range devices { tempDevice := deviceapi.Info{ Name: name, From 84e8866c83d70bf75be87e8edc802fe534b70834 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Thu, 5 Apr 2018 06:42:04 -0400 Subject: [PATCH 06/54] Using Metadata instead of MetaData --- glusterd2/commands/peers/addpeer.go | 14 +++++++------- glusterd2/commands/peers/editpeer.go | 16 ++++++++-------- glusterd2/commands/peers/getpeer.go | 2 +- glusterd2/commands/peers/getpeers.go | 2 +- glusterd2/peer/peer.go | 2 +- glusterd2/peer/self.go | 10 +++++----- pkg/api/peer_req_resp.go | 8 ++++---- plugins/device/api/resp.go | 2 +- plugins/device/rest.go | 2 +- plugins/device/store-utils.go | 6 +++--- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/glusterd2/commands/peers/addpeer.go b/glusterd2/commands/peers/addpeer.go index 0d3c96882..3820b552a 100644 --- a/glusterd2/commands/peers/addpeer.go +++ b/glusterd2/commands/peers/addpeer.go @@ -26,7 +26,7 @@ func addPeerHandler(w http.ResponseWriter, r *http.Request) { return } - for key := range req.MetaData { + for key := range req.Metadata { if strings.HasPrefix(key, "_") { logger.WithField("metadata-key", key).Error("Key names starting with '_' are restricted in metadata field") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Key names starting with '_' are restricted in metadata field") @@ -89,14 +89,14 @@ func addPeerHandler(w http.ResponseWriter, r *http.Request) { return } - if newpeer.MetaData == nil { - newpeer.MetaData = make(map[string]string) + if newpeer.Metadata == nil { + newpeer.Metadata = make(map[string]string) } if req.Zone != "" { - newpeer.MetaData["_zone"] = req.Zone + newpeer.Metadata["_zone"] = req.Zone } - for key, value := range req.MetaData { - newpeer.MetaData[key] = value + for key, value := range req.Metadata { + newpeer.Metadata[key] = value } err = peer.AddOrUpdatePeer(newpeer) @@ -118,6 +118,6 @@ func createPeerAddResp(p *peer.Peer) *api.PeerAddResp { Name: p.Name, PeerAddresses: p.PeerAddresses, ClientAddresses: p.ClientAddresses, - MetaData: p.MetaData, + Metadata: p.Metadata, } } diff --git a/glusterd2/commands/peers/editpeer.go b/glusterd2/commands/peers/editpeer.go index bce79cc3f..3e9031c39 100644 --- a/glusterd2/commands/peers/editpeer.go +++ b/glusterd2/commands/peers/editpeer.go @@ -31,7 +31,7 @@ func editPeer(w http.ResponseWriter, r *http.Request) { return } - for key := range req.MetaData { + for key := range req.Metadata { if strings.HasPrefix(key, "_") { logger.WithField("metadata-key", key).Error("Key names starting with '_' are restricted in metadata field") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Key names starting with '_' are restricted in metadata field") @@ -102,17 +102,17 @@ func txnPeerEdit(c transaction.TxnCtx) error { return err } - for k, v := range req.MetaData { - if peerInfo.MetaData != nil { - peerInfo.MetaData[k] = v + for k, v := range req.Metadata { + if peerInfo.Metadata != nil { + peerInfo.Metadata[k] = v } else { - peerInfo.MetaData = make(map[string]string) - peerInfo.MetaData[k] = v + peerInfo.Metadata = make(map[string]string) + peerInfo.Metadata[k] = v } } if req.Zone != "" { - peerInfo.MetaData["_zone"] = req.Zone + peerInfo.Metadata["_zone"] = req.Zone } err = peer.AddOrUpdatePeer(peerInfo) if err != nil { @@ -145,6 +145,6 @@ func createPeerEditResp(p *peer.Peer) *api.PeerEditResp { Name: p.Name, PeerAddresses: p.PeerAddresses, ClientAddresses: p.ClientAddresses, - MetaData: p.MetaData, + Metadata: p.Metadata, } } diff --git a/glusterd2/commands/peers/getpeer.go b/glusterd2/commands/peers/getpeer.go index 8a2ebdee6..0d4a93c7b 100644 --- a/glusterd2/commands/peers/getpeer.go +++ b/glusterd2/commands/peers/getpeer.go @@ -38,6 +38,6 @@ func createPeerGetResp(p *peer.Peer) *api.PeerGetResp { PeerAddresses: p.PeerAddresses, ClientAddresses: p.ClientAddresses, Online: store.Store.IsNodeAlive(p.ID), - MetaData: p.MetaData, + Metadata: p.Metadata, } } diff --git a/glusterd2/commands/peers/getpeers.go b/glusterd2/commands/peers/getpeers.go index 3eba4301d..19ffa5163 100644 --- a/glusterd2/commands/peers/getpeers.go +++ b/glusterd2/commands/peers/getpeers.go @@ -32,7 +32,7 @@ func createPeerListResp(peers []*peer.Peer) *api.PeerListResp { PeerAddresses: p.PeerAddresses, ClientAddresses: p.ClientAddresses, Online: store.Store.IsNodeAlive(p.ID), - MetaData: p.MetaData, + Metadata: p.Metadata, }) } diff --git a/glusterd2/peer/peer.go b/glusterd2/peer/peer.go index 2cf1505ff..407707a30 100644 --- a/glusterd2/peer/peer.go +++ b/glusterd2/peer/peer.go @@ -11,7 +11,7 @@ type Peer struct { Name string PeerAddresses []string ClientAddresses []string - MetaData map[string]string + Metadata map[string]string } // ETCDConfig represents the structure which holds the ETCD env variables & diff --git a/glusterd2/peer/self.go b/glusterd2/peer/self.go index f805ab5eb..643a18191 100644 --- a/glusterd2/peer/self.go +++ b/glusterd2/peer/self.go @@ -53,15 +53,15 @@ func AddSelfDetails() error { peerInfo, err := GetPeer(string(gdctx.MyUUID)) if err != nil { - if p.MetaData == nil { - p.MetaData = make(map[string]string) + if p.Metadata == nil { + p.Metadata = make(map[string]string) } - if p.MetaData["_zone"] == "" { - p.MetaData["_zone"] = p.ID.String() + if p.Metadata["_zone"] == "" { + p.Metadata["_zone"] = p.ID.String() } return AddOrUpdatePeer(p) } - peerInfo.MetaData = p.MetaData + peerInfo.Metadata = p.Metadata return nil } diff --git a/pkg/api/peer_req_resp.go b/pkg/api/peer_req_resp.go index 2fabff3a0..e8e4ea0aa 100644 --- a/pkg/api/peer_req_resp.go +++ b/pkg/api/peer_req_resp.go @@ -9,26 +9,26 @@ type Peer struct { PeerAddresses []string `json:"peer-addresses"` ClientAddresses []string `json:"client-addresses"` Online bool `json:"online"` - MetaData map[string]string `json:"metadata"` + Metadata map[string]string `json:"metadata"` } // PeerAddReq represents an incoming request to add a peer to the cluster type PeerAddReq struct { Addresses []string `json:"addresses"` Zone string `json:"zone,omitempty"` - MetaData map[string]string `json:"metadata,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` } // PeerEditReq represents an incoming request to edit metadata of peer type PeerEditReq struct { Zone string `json:"zone"` - MetaData map[string]string `json:"metadata"` + Metadata map[string]string `json:"metadata"` } // PeerAddResp is the success response sent to a PeerAddReq request type PeerAddResp Peer -// PeerEditResp is the success response sent to a MetaDataEditReq request +// PeerEditResp is the success response sent to a MetadataEditReq request type PeerEditResp Peer // PeerGetResp is the response sent for a peer get request diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index ecdc932ea..fccda3874 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -4,7 +4,7 @@ import ( "github.com/gluster/glusterd2/pkg/api" ) -// Info represents structure in which devices are to be store in Peer MetaData +// Info represents structure in which devices are to be store in Peer Metadata type Info struct { Name string `json:"name"` State string `json:"state"` diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 052a28e07..0cb8661b3 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -36,7 +36,7 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { return } - devices, err := CheckIfDeviceExist(req.Devices, peerInfo.MetaData["_devices"]) + devices, err := CheckIfDeviceExist(req.Devices, peerInfo.Metadata["_devices"]) if err != nil { logger.WithError(err).WithField("device", req.Devices).Error("Device already exist") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Devices already exist") diff --git a/plugins/device/store-utils.go b/plugins/device/store-utils.go index b80f516f8..c4578da9d 100644 --- a/plugins/device/store-utils.go +++ b/plugins/device/store-utils.go @@ -14,9 +14,9 @@ func GetDevices(peerID string) ([]deviceapi.Info, error) { if err != nil { return nil, err } - if len(peerInfo.MetaData["devices"]) > 0 { + if len(peerInfo.Metadata["devices"]) > 0 { var deviceInfo []deviceapi.Info - if err := json.Unmarshal([]byte(peerInfo.MetaData["devices"]), &deviceInfo); err != nil { + if err := json.Unmarshal([]byte(peerInfo.Metadata["devices"]), &deviceInfo); err != nil { return nil, err } return deviceInfo, nil @@ -73,7 +73,7 @@ func AddDevices(devices []deviceapi.Info, peerID string) error { if err != nil { return err } - peerInfo.MetaData["_devices"] = string(deviceJSON) + peerInfo.Metadata["_devices"] = string(deviceJSON) err = peer.AddOrUpdatePeer(peerInfo) if err != nil { return err From 2eabbe85d7cac648269354d5c0709aefab0b3a40 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 15 May 2018 13:43:15 +0530 Subject: [PATCH 07/54] added flag to set options during volume creation Signed-off-by: Madhu Rajanna --- glustercli/cmd/volume-create.go | 38 +++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/glustercli/cmd/volume-create.go b/glustercli/cmd/volume-create.go index b17147419..bd74113f5 100644 --- a/glustercli/cmd/volume-create.go +++ b/glustercli/cmd/volume-create.go @@ -23,6 +23,7 @@ var ( flagCreateForce bool flagCreateAdvOpts, flagCreateExpOpts, flagCreateDepOpts bool flagCreateThinArbiter string + flagCreateVolumeOptions []string volumeCreateCmd = &cobra.Command{ Use: "create []...", @@ -44,15 +45,11 @@ func init() { volumeCreateCmd.Flags().IntVar(&flagCreateRedundancyCount, "redundancy", 0, "Redundancy Count") volumeCreateCmd.Flags().StringVar(&flagCreateTransport, "transport", "tcp", "Transport") volumeCreateCmd.Flags().BoolVar(&flagCreateForce, "force", false, "Force") - - // XXX: These flags are currently hidden as the CLI does not yet support setting options during create. - // TODO: Make these visible once CLI gains support for setting options during create. + volumeCreateCmd.Flags().StringSliceVar(&flagCreateVolumeOptions, "options", []string{}, + "Volume options in the format option:value,option:value") volumeCreateCmd.Flags().BoolVar(&flagCreateAdvOpts, "advanced", false, "Allow advanced options") volumeCreateCmd.Flags().BoolVar(&flagCreateExpOpts, "experimental", false, "Allow experimental options") volumeCreateCmd.Flags().BoolVar(&flagCreateDepOpts, "deprecated", false, "Allow deprecated options") - volumeCreateCmd.Flags().MarkHidden("advanced") - volumeCreateCmd.Flags().MarkHidden("experimental") - volumeCreateCmd.Flags().MarkHidden("deprecated") volumeCmd.AddCommand(volumeCreateCmd) } @@ -100,10 +97,33 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { } } + options := make(map[string]string) + //set options + if len(flagCreateVolumeOptions) != 0 { + for _, opt := range flagCreateVolumeOptions { + + if len(strings.Split(opt, ":")) != 2 { + fmt.Println("Error setting volume options") + return + } + newopt := strings.Split(opt, ":") + //check if empty option or value + if len(strings.TrimSpace(newopt[0])) == 0 || len(strings.TrimSpace(newopt[1])) == 0 { + fmt.Println("Error setting volume options ,contains empty option or value") + return + } + options[newopt[0]] = newopt[1] + } + } + req := api.VolCreateReq{ - Name: volname, - Subvols: subvols, - Force: flagCreateForce, + Name: volname, + Subvols: subvols, + Force: flagCreateForce, + Options: options, + Advanced: flagCreateAdvOpts, + Experimental: flagCreateExpOpts, + Deprecated: flagCreateDepOpts, } // handle thin-arbiter From b5121e530737a84499c44709a3220976eccd2404 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 14 May 2018 10:50:43 -0400 Subject: [PATCH 08/54] filter volume list/info on basis of metadata keys and values --- e2e/volume_ops_test.go | 37 ++++++++++++++ glustercli/cmd/volume.go | 27 ++++++++-- glusterd2/commands/volumes/volume-list.go | 13 +++-- glusterd2/volume/store-utils.go | 60 +++++++++++++++++++++-- pkg/restclient/volume.go | 49 +++++++++++++++++- 5 files changed, 174 insertions(+), 12 deletions(-) diff --git a/e2e/volume_ops_test.go b/e2e/volume_ops_test.go index fad684bb2..e81201c2a 100644 --- a/e2e/volume_ops_test.go +++ b/e2e/volume_ops_test.go @@ -100,6 +100,9 @@ func testVolumeCreate(t *testing.T) { }, }, }, + Metadata: map[string]string{ + "owner": "gd2test", + }, Force: true, } _, err := client.VolumeCreate(createReq) @@ -148,6 +151,40 @@ func testVolumeStop(t *testing.T) { func testVolumeList(t *testing.T) { r := require.New(t) + var matchingQueries []map[string]string + var nonMatchingQueries []map[string]string + + matchingQueries = append(matchingQueries, map[string]string{ + "key": "owner", + "value": "gd2test", + }) + matchingQueries = append(matchingQueries, map[string]string{ + "key": "owner", + }) + matchingQueries = append(matchingQueries, map[string]string{ + "value": "gd2test", + }) + for _, filter := range matchingQueries { + volumes, err := client.Volumes("", filter) + r.Nil(err) + r.Len(volumes, 1) + } + + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "key": "owner", + "value": "gd2-test", + }) + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "key": "owners", + }) + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "value": "gd2tests", + }) + for _, filter := range nonMatchingQueries { + volumes, err := client.Volumes("", filter) + r.Nil(err) + r.Len(volumes, 0) + } volumes, err := client.Volumes("") r.Nil(err) diff --git a/glustercli/cmd/volume.go b/glustercli/cmd/volume.go index f255bbc67..9a57ef71f 100644 --- a/glustercli/cmd/volume.go +++ b/glustercli/cmd/volume.go @@ -41,6 +41,10 @@ var ( flagExpandCmdReplicaCount int flagExpandCmdForce bool + // Filter Volume Info/List command flags + flagCmdFilterKey string + flagCmdFilterValue string + // Edit Command Flags flagCmdMetadataKey string flagCmdMetadataValue string @@ -62,10 +66,14 @@ func init() { volumeCmd.AddCommand(volumeGetCmd) volumeCmd.AddCommand(volumeResetCmd) + volumeInfoCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter Key") + volumeInfoCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter Value") volumeCmd.AddCommand(volumeInfoCmd) volumeCmd.AddCommand(volumeStatusCmd) + volumeListCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter Key") + volumeListCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter Value") volumeCmd.AddCommand(volumeListCmd) // Volume Expand @@ -281,8 +289,21 @@ func volumeInfoHandler2(cmd *cobra.Command, isInfo bool) error { volname = cmd.Flags().Args()[0] } if volname == "" { - vols, err = client.Volumes("") + if flagCmdFilterKey == "" && flagCmdFilterValue == "" { + vols, err = client.Volumes("") + } else if flagCmdFilterKey != "" && flagCmdFilterValue == "" { + vols, err = client.Volumes("", map[string]string{"key": flagCmdFilterKey}) + } else if flagCmdFilterKey == "" && flagCmdFilterValue != "" { + vols, err = client.Volumes("", map[string]string{"value": flagCmdFilterValue}) + } else if flagCmdFilterKey != "" && flagCmdFilterValue != "" { + vols, err = client.Volumes("", map[string]string{"key": flagCmdFilterKey, + "value": flagCmdFilterValue, + }) + } } else { + if flagCmdFilterKey != "" || flagCmdFilterValue != "" { + return errors.New("Invalid command. Cannot give filter arguments when providing volname") + } vols, err = client.Volumes(volname) } @@ -306,7 +327,7 @@ func volumeInfoHandler2(cmd *cobra.Command, isInfo bool) error { } var volumeInfoCmd = &cobra.Command{ - Use: "info", + Use: "info [ |--key |--value |--key --value ]", Short: helpVolumeInfoCmd, Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { @@ -323,7 +344,7 @@ var volumeInfoCmd = &cobra.Command{ } var volumeListCmd = &cobra.Command{ - Use: "list", + Use: "list [--key |--value |--key --value ]", Short: helpVolumeListCmd, Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { diff --git a/glusterd2/commands/volumes/volume-list.go b/glusterd2/commands/volumes/volume-list.go index 4b84a5455..06deecda7 100644 --- a/glusterd2/commands/volumes/volume-list.go +++ b/glusterd2/commands/volumes/volume-list.go @@ -11,12 +11,19 @@ import ( func volumeListHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - - volumes, err := volume.GetVolumes() + keys, keyFound := r.URL.Query()["key"] + values, valueFound := r.URL.Query()["value"] + filterParams := make(map[string]string) + if keyFound { + filterParams["key"] = keys[0] + } + if valueFound { + filterParams["value"] = values[0] + } + volumes, err := volume.GetVolumes(filterParams) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) } - resp := createVolumeListResp(volumes) restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) } diff --git a/glusterd2/volume/store-utils.go b/glusterd2/volume/store-utils.go index 609f7681c..99b043169 100644 --- a/glusterd2/volume/store-utils.go +++ b/glusterd2/volume/store-utils.go @@ -16,6 +16,17 @@ const ( volumePrefix string = "volumes/" ) +// metadataFilter is a filter type +type metadataFilter uint32 + +// GetVolumes Filter Types +const ( + NoKeyAndValue metadataFilter = iota + OnlyKey + OnlyValue + KeyAndValue +) + var ( // AddOrUpdateVolumeFunc marshals to volume object and passes to store to add/update AddOrUpdateVolumeFunc = AddOrUpdateVolume @@ -91,17 +102,38 @@ func GetVolumesList() (map[string]uuid.UUID, error) { return volumes, nil } +// getFilterType return the filter type for volume list/info +func getFilterType(filterParams map[string]string) metadataFilter { + _, key := filterParams["key"] + _, value := filterParams["value"] + if key && !value { + return OnlyKey + } else if value && !key { + return OnlyValue + } else if value && key { + return KeyAndValue + } + return NoKeyAndValue +} + //GetVolumes retrives the json objects from the store and converts them into //respective volinfo objects -func GetVolumes() ([]*Volinfo, error) { +func GetVolumes(filterParams ...map[string]string) ([]*Volinfo, error) { resp, e := store.Store.Get(context.TODO(), volumePrefix, clientv3.WithPrefix()) if e != nil { return nil, e } - volumes := make([]*Volinfo, len(resp.Kvs)) + var filterType metadataFilter + if len(filterParams) == 0 { + filterType = NoKeyAndValue + } else { + filterType = getFilterType(filterParams[0]) + } + + var volumes []*Volinfo - for i, kv := range resp.Kvs { + for _, kv := range resp.Kvs { var vol Volinfo if err := json.Unmarshal(kv.Value, &vol); err != nil { @@ -111,8 +143,28 @@ func GetVolumes() ([]*Volinfo, error) { }).Error("Failed to unmarshal volume") continue } + switch filterType { + + case OnlyKey: + if _, keyFound := vol.Metadata[filterParams[0]["key"]]; keyFound { + volumes = append(volumes, &vol) + } + case OnlyValue: + for _, value := range vol.Metadata { + if value == filterParams[0]["value"] { + volumes = append(volumes, &vol) + } + } + case KeyAndValue: + if value, keyFound := vol.Metadata[filterParams[0]["key"]]; keyFound { + if value == filterParams[0]["value"] { + volumes = append(volumes, &vol) + } + } + default: + volumes = append(volumes, &vol) - volumes[i] = &vol + } } return volumes, nil diff --git a/pkg/restclient/volume.go b/pkg/restclient/volume.go index 568ab5bb2..988684000 100644 --- a/pkg/restclient/volume.go +++ b/pkg/restclient/volume.go @@ -3,10 +3,22 @@ package restclient import ( "fmt" "net/http" + "net/url" "github.com/gluster/glusterd2/pkg/api" ) +// metadataFilter is a filter type +type metadataFilter uint32 + +// GetVolumes Filter Types +const ( + NoKeyAndValue metadataFilter = iota + OnlyKey + OnlyValue + KeyAndValue +) + // VolumeCreate creates Gluster Volume func (c *Client) VolumeCreate(req api.VolCreateReq) (api.VolumeCreateResp, error) { var vol api.VolumeCreateResp @@ -14,11 +26,44 @@ func (c *Client) VolumeCreate(req api.VolCreateReq) (api.VolumeCreateResp, error return vol, err } +// getFilterType return the filter type for volume list/info +func getFilterType(filterParams map[string]string) metadataFilter { + _, key := filterParams["key"] + _, value := filterParams["value"] + if key && !value { + return OnlyKey + } else if value && !key { + return OnlyValue + } else if value && key { + return KeyAndValue + } + return NoKeyAndValue +} + +// getQueryString returns the query string for filtering volumes +func getQueryString(filterParam map[string]string) string { + filterType := getFilterType(filterParam) + var queryString string + switch filterType { + case OnlyKey: + queryString = fmt.Sprintf("?key=%s", url.QueryEscape(filterParam["key"])) + case OnlyValue: + queryString = fmt.Sprintf("?value=%s", url.QueryEscape(filterParam["value"])) + case KeyAndValue: + queryString = fmt.Sprintf("?key=%s&value=%s", url.QueryEscape(filterParam["key"]), url.QueryEscape(filterParam["value"])) + } + return queryString +} + // Volumes returns list of all volumes -func (c *Client) Volumes(volname string) (api.VolumeListResp, error) { +func (c *Client) Volumes(volname string, filterParams ...map[string]string) (api.VolumeListResp, error) { if volname == "" { var vols api.VolumeListResp - url := fmt.Sprintf("/v1/volumes") + var queryString string + if len(filterParams) > 0 { + queryString = getQueryString(filterParams[0]) + } + url := fmt.Sprintf("/v1/volumes%s", queryString) err := c.get(url, nil, http.StatusOK, &vols) return vols, err } From dc0d7a52d54811ba7684ea68bc207d7188a0bd6a Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 16 May 2018 16:02:13 +0530 Subject: [PATCH 09/54] Remove ResponseHeaderTimeout ...that was set to 3 seconds. ResponseHeaderTimeout specifies the amount of time to wait for a server's response headers after fully writing the request. This time does not include the time to read the response body. Instead, set 'Timeout' field of http.Client to 30s which includes TCP connection time and also time taken to read the response body. Regardless of how many nodes (even 100s) make up a volume, 30 seconds is a large enough timeout for completion of a request. Note: * We can expose --timeout as a CLI option in future if deemed required. * We may also introduce async APIs for long running tasks such as dynamic volume provisioning. Signed-off-by: Prashanth Pai --- pkg/restclient/common.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/restclient/common.go b/pkg/restclient/common.go index 949881cc8..0a368bf79 100644 --- a/pkg/restclient/common.go +++ b/pkg/restclient/common.go @@ -16,8 +16,9 @@ import ( "github.com/dgrijalva/jwt-go" ) -var ( +const ( expireSeconds = 120 + clientTimeout = 30 ) // Client represents Glusterd2 REST Client @@ -47,7 +48,7 @@ func parseHTTPError(jsonData []byte) string { func getAuthToken(username string, password string) string { // Create the Claims claims := &jwt.StandardClaims{ - ExpiresAt: time.Now().Add(time.Second * time.Duration(expireSeconds)).Unix(), + ExpiresAt: time.Now().Add(time.Second * expireSeconds).Unix(), Issuer: username, } token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) @@ -93,6 +94,7 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod } req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") + req.Close = true // Set Authorization if username and password is not empty string if c.username != "" && c.password != "" { @@ -100,9 +102,8 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod } tr := &http.Transport{ - DisableCompression: true, - DisableKeepAlives: true, - ResponseHeaderTimeout: 3 * time.Second, + DisableCompression: true, + DisableKeepAlives: true, } if c.cacert != "" || c.insecure { @@ -120,7 +121,9 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod } } - client := &http.Client{Transport: tr} + client := &http.Client{ + Transport: tr, + Timeout: clientTimeout * time.Second} resp, err := client.Do(req) if err != nil { From 08908e2b5a2290856a9aabb696268ae271542fda Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Thu, 17 May 2018 16:59:01 +0530 Subject: [PATCH 10/54] http: Introduce server timeouts A malicious client can open many connections and make the HTTP server run out of file descriptors by intentionally being slow in sending request headers or body. This PR adds timeouts on the http server and also sets maximum header size to 8KB (default in apache). Signed-off-by: Prashanth Pai --- glusterd2/servers/rest/rest.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/glusterd2/servers/rest/rest.go b/glusterd2/servers/rest/rest.go index 8d093d528..32b35070b 100644 --- a/glusterd2/servers/rest/rest.go +++ b/glusterd2/servers/rest/rest.go @@ -21,6 +21,12 @@ import ( config "github.com/spf13/viper" ) +const ( + httpReadTimeout = 10 + httpWriteTimeout = 30 + maxHeaderBytes = 1 << 13 // 8KB +) + // GDRest is the GlusterD Rest server type GDRest struct { Routes *mux.Router @@ -50,7 +56,11 @@ func NewMuxed(m cmux.CMux) *GDRest { rest := &GDRest{ Routes: mux.NewRouter(), - server: &http.Server{}, + server: &http.Server{ + ReadTimeout: httpReadTimeout * time.Second, + WriteTimeout: httpWriteTimeout * time.Second, + MaxHeaderBytes: maxHeaderBytes, + }, stopCh: make(chan struct{}), } From 093806fca965c06d0589e1fda51a55e2f82a1267 Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Fri, 18 May 2018 15:10:46 +0530 Subject: [PATCH 11/54] bitrot: set advanced flag to bitrot volume options Fixes: #767 and #766 Signed-off-by: Aravinda VK --- glustercli/cmd/bitrot.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/glustercli/cmd/bitrot.go b/glustercli/cmd/bitrot.go index 898d43911..5dbf1ac05 100644 --- a/glustercli/cmd/bitrot.go +++ b/glustercli/cmd/bitrot.go @@ -100,6 +100,8 @@ var bitrotScrubThrottleCmd = &cobra.Command{ option = append(option, "bit-rot.scrub-throttle") option = append(option, args[1]) + // Set Option set flag to advanced + flagSetAdv = true err := volumeOptionJSONHandler(cmd, volname, option) if err != nil { if verbose { @@ -125,6 +127,8 @@ var bitrotScrubFrequencyCmd = &cobra.Command{ option = append(option, "bit-rot.scrub-freq") option = append(option, args[1]) + // Set Option set flag to advanced + flagSetAdv = true err := volumeOptionJSONHandler(cmd, volname, option) if err != nil { if verbose { @@ -152,6 +156,9 @@ var bitrotScrubCmd = &cobra.Command{ case scrubPause, scrubResume: option = append(option, "bit-rot.scrub-state") option = append(option, args[1]) + + // Set Option set flag to advanced + flagSetAdv = true err := volumeOptionJSONHandler(cmd, volname, option) if err != nil { if verbose { From 3ed713698454a3eb7c3df8b6bdf5858efefa2c93 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Mon, 21 May 2018 00:53:24 -0400 Subject: [PATCH 12/54] Renaming commands folder to utils --- glusterd2/commands/peers/addpeer.go | 3 +- pkg/errors/error.go | 1 + plugins/device/deviceutils/utils.go | 43 +++++++++++++++++++++++++++++ plugins/device/transaction.go | 24 ++++++++++++---- 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 plugins/device/deviceutils/utils.go diff --git a/glusterd2/commands/peers/addpeer.go b/glusterd2/commands/peers/addpeer.go index 9cc18a983..d5b11b24b 100644 --- a/glusterd2/commands/peers/addpeer.go +++ b/glusterd2/commands/peers/addpeer.go @@ -28,8 +28,7 @@ func addPeerHandler(w http.ResponseWriter, r *http.Request) { for key := range req.Metadata { if strings.HasPrefix(key, "_") { - logger.WithField("metadata-key", key).Error("Key names starting with '_' are restricted in metadata field") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Key names starting with '_' are restricted in metadata field") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, errors.ErrRestrictedKeyFound) return } } diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 525550326..50020f776 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -40,4 +40,5 @@ var ( ErrUnmarshallFailed = errors.New("failed to unmarshall from json") ErrClusterNotFound = errors.New("Cluster instance not found in store") ErrDuplicateBrickPath = errors.New("Duplicate brick entry") + ErrRestrictedKeyFound = errors.New("Key names starting with '_' are restricted in metadata field") ) diff --git a/plugins/device/deviceutils/utils.go b/plugins/device/deviceutils/utils.go new file mode 100644 index 000000000..105259d4b --- /dev/null +++ b/plugins/device/deviceutils/utils.go @@ -0,0 +1,43 @@ +package deviceutils + +import ( + "os/exec" +) + +// +func CreatePV(device string) error { + + pvcreateCmd := exec.Command("pvcreate", "--metadatasize=128M", "--dataalignment=256K", device) + if err := pvcreateCmd.Run(); err != nil { + return err + } + return nil +} + +// +func CreateVG(device string, vgName string) error { + + vgcreateCmd := exec.Command("vgcreate", vgName, device) + if err := vgcreateCmd.Run(); err != nil { + return err + } + return nil +} + +// +func RemoveVG(vgName string) error { + vgremoveCmd := exec.Command("vgremove", vgName) + if err := vgremoveCmd.Run(); err != nil { + return err + } + return nil +} + +// +func RemovePV(device string) error { + pvremoveCmd := exec.Command("pvremove", device) + if err := pvremoveCmd.Run(); err != nil { + return err + } + return nil +} diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index 8bb95e6ab..e6d743c63 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -1,9 +1,11 @@ package device import ( + "strings" + "github.com/gluster/glusterd2/glusterd2/transaction" deviceapi "github.com/gluster/glusterd2/plugins/device/api" - "github.com/gluster/glusterd2/plugins/device/cmdexec" + "github.com/gluster/glusterd2/plugins/device/deviceutils" ) func txnPrepareDevice(c transaction.TxnCtx) error { @@ -27,18 +29,30 @@ func txnPrepareDevice(c transaction.TxnCtx) error { deviceList = append(deviceList, tempDevice) } + var failedDevice []string + var successDevice []deviceapi.Info for index, device := range deviceList { - err := cmdexec.DeviceSetup(c, device.Name) + err := deviceutils.CreatePV(device.Name) if err != nil { - c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to setup device, setting device status to 'DeviceFailed'") - deviceList[index].State = deviceapi.DeviceFailed + c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to create physical volume") continue } + vgName := strings.Replace("vg"+device.Name, "/", "-", -1) + err = deviceutils.CreateVG(device.Name, vgName) + if err != nil { + c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to create volume group") + err = deviceutils.RemovePV(device.Name) + if err != nil { + c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to remove physical volume") + failedDevice = append(failedDevice, device.Name) + } + } c.Logger().WithError(err).WithField("device", device.Name).Error("Setup device successful, setting device status to 'DeviceEnabled'") deviceList[index].State = deviceapi.DeviceEnabled + successDevice = append(successDevice, deviceList[index]) } - err := AddDevices(deviceList, peerID) + err := AddDevices(successDevice, peerID) if err != nil { c.Logger().WithError(err).Error("Couldn't add deviceinfo to store") return err From 91b0a8e78ab17af6f44feeea145a95a7e2c09191 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 21 May 2018 12:38:40 +0530 Subject: [PATCH 13/54] validate volume name It is necessary to validate volume name before volume creation Signed-off-by: Madhu Rajanna --- e2e/volume_ops_test.go | 5 +++++ glusterd2/commands/volumes/volume-create.go | 11 +++++++++++ pkg/errors/error.go | 1 + 3 files changed, 17 insertions(+) diff --git a/e2e/volume_ops_test.go b/e2e/volume_ops_test.go index e81201c2a..087586202 100644 --- a/e2e/volume_ops_test.go +++ b/e2e/volume_ops_test.go @@ -107,6 +107,11 @@ func testVolumeCreate(t *testing.T) { } _, err := client.VolumeCreate(createReq) r.Nil(err) + + //invalid volume name + createReq.Name = "##@@#@!#@!!@#" + _, err = client.VolumeCreate(createReq) + r.NotNil(err) } func testVolumeExpand(t *testing.T) { diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index c8806c691..84c718bfb 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -4,6 +4,7 @@ import ( "errors" "net/http" "path/filepath" + "regexp" "github.com/gluster/glusterd2/glusterd2/events" "github.com/gluster/glusterd2/glusterd2/gdctx" @@ -16,8 +17,18 @@ import ( "github.com/pborman/uuid" ) +var ( + reg = regexp.MustCompile("^[a-zA-Z0-9_-]+$") +) + func validateVolCreateReq(req *api.VolCreateReq) error { + valid := reg.MatchString(req.Name) + + if !valid { + return gderrors.ErrInvalidVolName + } + if req.Name == "" { return gderrors.ErrEmptyVolName } diff --git a/pkg/errors/error.go b/pkg/errors/error.go index a63771995..28bc3a7af 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -12,6 +12,7 @@ var ( ErrPeerNotFound = errors.New("peer not found") ErrJSONParsingFailed = errors.New("unable to parse the request") ErrEmptyVolName = errors.New("volume name is empty") + ErrInvalidVolName = errors.New("invalid volume name") ErrEmptyBrickList = errors.New("brick list is empty") ErrInvalidBrickPath = errors.New("invalid brick path, brick path should be in host: format") ErrVolExists = errors.New("volume already exists") From 01e5f9d7981e70c6aeebcaf72b9ef675f880945b Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 21 May 2018 13:40:16 +0530 Subject: [PATCH 14/54] regex validation will handle empty volname, removed validation for empty volume name Signed-off-by: Madhu Rajanna --- glusterd2/commands/volumes/volume-create.go | 4 ---- pkg/errors/error.go | 1 - 2 files changed, 5 deletions(-) diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index 84c718bfb..5881b328a 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -29,10 +29,6 @@ func validateVolCreateReq(req *api.VolCreateReq) error { return gderrors.ErrInvalidVolName } - if req.Name == "" { - return gderrors.ErrEmptyVolName - } - if req.Transport != "" && req.Transport != "tcp" && req.Transport != "rdma" { return errors.New("Invalid transport. Supported values: tcp or rdma") } diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 28bc3a7af..5a6578262 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -11,7 +11,6 @@ var ( ErrVolNotStarted = errors.New("volume not started") ErrPeerNotFound = errors.New("peer not found") ErrJSONParsingFailed = errors.New("unable to parse the request") - ErrEmptyVolName = errors.New("volume name is empty") ErrInvalidVolName = errors.New("invalid volume name") ErrEmptyBrickList = errors.New("brick list is empty") ErrInvalidBrickPath = errors.New("invalid brick path, brick path should be in host: format") From 7a3af99b966d596daacc756d13faa3933dcbfbe0 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Mon, 21 May 2018 06:04:14 -0400 Subject: [PATCH 15/54] Using global error message --- glusterd2/commands/peers/editpeer.go | 4 ++-- glusterd2/commands/volumes/volume-edit.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/glusterd2/commands/peers/editpeer.go b/glusterd2/commands/peers/editpeer.go index 8d9d57433..b73beb5d8 100644 --- a/glusterd2/commands/peers/editpeer.go +++ b/glusterd2/commands/peers/editpeer.go @@ -34,8 +34,8 @@ func editPeer(w http.ResponseWriter, r *http.Request) { for key := range req.MetaData { if strings.HasPrefix(key, "_") { - logger.WithField("metadata-key", key).Error("Key names starting with '_' are restricted in metadata field") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Key names starting with '_' are restricted in metadata field") + logger.WithField("metadata-key", key).Error(errors.ErrRestrictedKeyFound) + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrRestrictedKeyFound) return } } diff --git a/glusterd2/commands/volumes/volume-edit.go b/glusterd2/commands/volumes/volume-edit.go index c6fe643a1..e8dd0a1a9 100644 --- a/glusterd2/commands/volumes/volume-edit.go +++ b/glusterd2/commands/volumes/volume-edit.go @@ -53,8 +53,8 @@ func volumeEditHandler(w http.ResponseWriter, r *http.Request) { for key, value := range req.Metadata { if strings.HasPrefix(key, "_") { - logger.WithField("key", key).Error("Key names starting with '_' are restricted in metadata field") - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Key names starting with '_' are restricted in metadata field") + logger.WithField("key", key).Error(errors.ErrRestrictedKeyFound) + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrRestrictedKeyFound) return } if req.DeleteMetadata { From fec67454e1ab9f01a43dc0761102d602b95be7f1 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 21 May 2018 13:17:45 +0530 Subject: [PATCH 16/54] Add script to build nightly-rpms --- Makefile | 4 +- extras/nightly-rpms.sh | 81 ++++++++++++++++++++++++++++++++++++++ extras/rpms/glusterd2.spec | 4 +- scripts/dist.sh | 14 ++++--- 4 files changed, 94 insertions(+), 9 deletions(-) create mode 100755 extras/nightly-rpms.sh diff --git a/Makefile b/Makefile index b96649bc8..7c2315815 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ release: build @./scripts/release.sh dist: - @./scripts/dist.sh + @DISTDIR=$(DISTDIR) SIGN=$(SIGN) ./scripts/dist.sh dist-vendor: vendor-install - @VENDOR=yes ./scripts/dist.sh + @VENDOR=yes DISTDIR=$(DISTDIR) SIGN=$(SIGN) ./scripts/dist.sh diff --git a/extras/nightly-rpms.sh b/extras/nightly-rpms.sh new file mode 100755 index 000000000..e011cdad6 --- /dev/null +++ b/extras/nightly-rpms.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# This scripts builds RPMs from the current git head. +# The script needs be run from the root of the repository +# NOTE: RPMs are built only for EL7 (CentOS7) distributions. + +set -e + +## +## Set up build environment +## +RESULTDIR=${RESULTDIR:-$PWD/rpms} +BUILDDIR=$PWD/$(mktemp -d nightlyrpmXXXXXX) + +BASEDIR=$(dirname "$0") +GD2CLONE=$(realpath "$BASEDIR/..") + +yum -y install make mock rpm-build golang + +export GOPATH=$BUILDDIR/go +mkdir -p "$GOPATH"/{bin,pkg,src} +export PATH=$GOPATH/bin:$PATH + +GD2SRC=$GOPATH/src/github.com/gluster/glusterd2 +mkdir -p "$GOPATH/src/github.com/gluster" +ln -s "$GD2CLONE" "$GD2SRC" + +"$GD2SRC/scripts/install-reqs.sh" + +## +## Prepare GD2 archives and specfile for building RPMs +## +pushd "$GD2SRC" + +VERSION=$(./scripts/pkg-version --version) +RELEASE=$(./scripts/pkg-version --release) +FULL_VERSION=$(./scripts/pkg-version --full) + +# Create a vendored dist archive +DISTDIR=$BUILDDIR SIGN=no make dist-vendor + +# Copy over specfile to the BUILDDIR and modify it to use the current Git HEAD versions +cp ./extras/rpms/* "$BUILDDIR" + +popd #GD2SRC + +pushd "$BUILDDIR" + +DISTARCHIVE="glusterd2-$FULL_VERSION-vendor.tar.xz" +SPEC=glusterd2.spec +sed -i -E " +# Use bundled always +s/with_bundled 0/with_bundled 1/; +# Replace version with HEAD version +s/^Version:[[:space:]]+([0-9]+\\.)*[0-9]+$/Version: $VERSION/; +# Replace release with proper release +s/^Release:[[:space:]]+.*%\\{\\?dist\\}/Release: $RELEASE%{?dist}/; +# Replace Source0 with generated archive +s/^Source0:[[:space:]]+.*-vendor.tar.xz/Source0: $DISTARCHIVE/; +# Change prep setup line to use correct release +s/^(%setup -q -n %\\{name\\}-v%\\{version\\}-)(0)/\\1$RELEASE/; +" $SPEC + +## +## Build the RPMs +## + +# Create SRPM +mkdir -p rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} +cp "$DISTARCHIVE" glusterd2-logrotate rpmbuild/SOURCES +cp $SPEC rpmbuild/SPECS +SRPM=$(rpmbuild --define "_topdir $PWD/rpmbuild" -bs rpmbuild/SPECS/$SPEC | cut -d\ -f2) + +# Build RPM from SRPM using mock +mkdir -p "$RESULTDIR" +mock -r epel-7-x86_64 --resultdir="$RESULTDIR" --rebuild "$SRPM" + +popd #BUILDDIR + +## Cleanup +rm -rf "$BUILDDIR" diff --git a/extras/rpms/glusterd2.spec b/extras/rpms/glusterd2.spec index e1a139b8a..7fba4faa0 100644 --- a/extras/rpms/glusterd2.spec +++ b/extras/rpms/glusterd2.spec @@ -24,8 +24,8 @@ %global gd2make %{__make} PREFIX=%{_prefix} EXEC_PREFIX=%{_exec_prefix} BINDIR=%{_bindir} SBINDIR=%{_sbindir} DATADIR=%{_datadir} LOCALSTATEDIR=%{_sharedstatedir} LOGDIR=%{_localstatedir}/log SYSCONFDIR=%{_sysconfdir} FASTBUILD=off Name: %{repo} -Version: 4.0.0 -Release: 2%{?dist} +Version: 4.1.0 +Release: dev%{?dist} Summary: The GlusterFS management daemon (preview) License: GPLv2 or LGPLv3+ URL: https://%{provider_prefix} diff --git a/scripts/dist.sh b/scripts/dist.sh index d6e844b9a..acb2a3fa4 100755 --- a/scripts/dist.sh +++ b/scripts/dist.sh @@ -4,7 +4,8 @@ # This should only be called from the root of the GD2 repo VENDOR=${VENDOR:-no} -OUTDIR=${1:-.} +OUTDIR=${DISTDIR:-.} +SIGN=${SIGN:-yes} VERSION=$("$(dirname "$0")/pkg-version" --full) @@ -45,10 +46,13 @@ echo "Created dist archive $ARCHIVE" # Sign the generated archive -echo "Signing dist archive" -gpg --armor --output "$SIGNFILE" --detach-sign "$ARCHIVE" || exit 1 -echo "Signed dist archive, signature in $SIGNFILE" - +case $SIGN in + yes|y|Y) + echo "Signing dist archive" + gpg --armor --output "$SIGNFILE" --detach-sign "$ARCHIVE" || exit 1 + echo "Signed dist archive, signature in $SIGNFILE" + ;; +esac # Remove the VERSION file, it is no longer needed and would harm normal builds rm VERSION From a565046e6f2ebbf18556838db7fc331203f3db58 Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Fri, 18 May 2018 14:33:47 +0530 Subject: [PATCH 17/54] Configuration improvements If `PREFIX` is `/usr` then `logdir`, `rundir` and `localstatedir` should not contain the prefix. **Global default configuration**: Based on the `--prefix` used during installation/make global config should be installed in `$PREFIX/etc/glusterfs/glusterd2.toml`. All the default values from code should be moved to this file. **Removed logic to pick config file from current directory**: Since Glusterd2 will be run as daemon or background process, automatically picking config file from current directory will add confusion to users. For development, if global config file not exists then it loads config file from the current directory. **Config file as argument**: Glusterd2 applies global configurations first and then applies this configuration to overwrite the values if any. Signed-off-by: Aravinda VK --- .gitignore | 2 + Makefile | 6 ++- e2e/config/1.toml | 3 ++ e2e/config/2.toml | 3 ++ e2e/config/3.toml | 3 ++ e2e/config/4.toml | 3 ++ extras/make/paths.mk | 13 ++++-- glusterd2/config.go | 74 +++++++++++++++++----------------- scripts/gen-gd2conf.sh | 15 ++++++- scripts/prepare_path_config.sh | 13 ++++++ 10 files changed, 90 insertions(+), 45 deletions(-) create mode 100755 scripts/prepare_path_config.sh diff --git a/.gitignore b/.gitignore index c5e6301d9..911d8b86f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ endpoints.json generate-doc # Ignore vi backup files *~ +# Generated by Make +glusterd2/paths_config.go diff --git a/Makefile b/Makefile index b96649bc8..d9484012f 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,12 @@ check-reqs: @./scripts/check-reqs.sh @echo -$(GD2_BIN): $(GD2_BUILD) +$(GD2_BIN): $(GD2_BUILD) gd2conf $(GD2_BUILD): + @PREFIX=$(PREFIX) BASE_PREFIX=$(BASE_PREFIX) EXEC_PREFIX=$(EXEC_PREFIX) \ + BINDIR=$(BINDIR) SBINDIR=$(SBINDIR) DATADIR=$(DATADIR) \ + LOCALSTATEDIR=$(LOCALSTATEDIR) LOGDIR=$(LOGDIR) \ + SYSCONFDIR=$(SYSCONFDIR) ./scripts/prepare_path_config.sh glusterd2 @PLUGINS=$(PLUGINS) FASTBUILD=$(FASTBUILD) ./scripts/build.sh glusterd2 @echo diff --git a/e2e/config/1.toml b/e2e/config/1.toml index f31114993..10383c008 100644 --- a/e2e/config/1.toml +++ b/e2e/config/1.toml @@ -1,4 +1,7 @@ workdir = "w1" +localstatedir = "w1" +logdir = "w1/log" +loglevel = "DEBUG" rundir = "w1/run/gluster" logfile = "w1.log" peeraddress = "127.0.0.1:24008" diff --git a/e2e/config/2.toml b/e2e/config/2.toml index 0844066db..e4d08b031 100644 --- a/e2e/config/2.toml +++ b/e2e/config/2.toml @@ -1,4 +1,7 @@ workdir = "w2" +localstatedir = "w2" +logdir = "w2/log" +loglevel = "DEBUG" rundir = "w2/run/gluster" logfile = "w2.log" peeraddress = "127.0.0.1:23008" diff --git a/e2e/config/3.toml b/e2e/config/3.toml index 29947531d..a50781624 100644 --- a/e2e/config/3.toml +++ b/e2e/config/3.toml @@ -1,4 +1,7 @@ workdir = "w3" +localstatedir = "w3" +logdir = "w3/log" +loglevel = "DEBUG" rundir = "w3/run/gluster" logfile = "w3.log" peeraddress = "127.0.0.1:22008" diff --git a/e2e/config/4.toml b/e2e/config/4.toml index 3b9cf5817..446e96cf3 100644 --- a/e2e/config/4.toml +++ b/e2e/config/4.toml @@ -1,4 +1,7 @@ workdir = "w4" +localstatedir = "w4" +logdir = "w4/log" +loglevel = "DEBUG" rundir = "w4/run/gluster" logfile = "w4.log" peeraddress = "127.0.0.1:21008" diff --git a/extras/make/paths.mk b/extras/make/paths.mk index 5865289fd..07db1dc8e 100644 --- a/extras/make/paths.mk +++ b/extras/make/paths.mk @@ -5,14 +5,19 @@ # https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros PREFIX ?= /usr/local + +BASE_PREFIX = $(PREFIX) +ifeq ($(PREFIX), /usr) +BASE_PREFIX = "" +endif + EXEC_PREFIX ?= $(PREFIX) BINDIR ?= $(EXEC_PREFIX)/bin SBINDIR ?= $(EXEC_PREFIX)/sbin DATADIR ?= $(PREFIX)/share -LOCALSTATEDIR ?= $(PREFIX)/var/lib -LOGDIR ?= $(PREFIX)/var/log - -SYSCONFDIR ?= $(PREFIX)/etc +LOCALSTATEDIR ?= $(BASE_PREFIX)/var/lib +LOGDIR ?= $(BASE_PREFIX)/var/log +SYSCONFDIR ?= $(BASE_PREFIX)/etc diff --git a/glusterd2/config.go b/glusterd2/config.go index cf92a5e80..95d3d7153 100644 --- a/glusterd2/config.go +++ b/glusterd2/config.go @@ -7,6 +7,7 @@ import ( "net" "os" "path" + "path/filepath" "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/store" @@ -23,27 +24,15 @@ var ( ) const ( - defaultLogLevel = "debug" - defaultClientAddress = ":24007" - defaultPeerAddress = ":24008" - defaultConfName = "glusterd2" - defaultRunDir = "/var/run/gluster" -) - -// Slices,Arrays cannot be constants :( -var ( - defaultConfPaths = []string{ - "/etc/glusterd2", - "/usr/local/etc/glusterd2", - ".", - } + defaultLogLevel = "debug" + defaultConfName = "glusterd2" ) // parseFlags sets up the flags and parses them, this needs to be called before any other operation func parseFlags() { flag.String("workdir", "", "Working directory for GlusterD. (default: current directory)") flag.String("localstatedir", "", "Directory to store local state information. (default: workdir)") - flag.String("rundir", defaultRunDir, "Directory to store runtime data.") + flag.String("rundir", "", "Directory to store runtime data.") flag.String("config", "", "Configuration file for GlusterD. By default looks for glusterd2.toml in [/usr/local]/etc/glusterd2 and current working directory.") flag.String(logging.DirFlag, "", logging.DirHelp+" (default: workdir/log)") @@ -53,8 +42,8 @@ func parseFlags() { // TODO: Change default to false (disabled) in future. flag.Bool("statedump", true, "Enable /statedump endpoint for metrics.") - flag.String("clientaddress", defaultClientAddress, "Address to bind the REST service.") - flag.String("peeraddress", defaultPeerAddress, "Address to bind the inter glusterd2 RPC service.") + flag.String("clientaddress", "", "Address to bind the REST service.") + flag.String("peeraddress", "", "Address to bind the inter glusterd2 RPC service.") // TODO: SSL/TLS is currently only implemented for REST interface flag.String("cert-file", "", "Certificate used for SSL/TLS connections from clients to glusterd2.") @@ -96,9 +85,6 @@ func setDefaults() error { config.SetDefault("pidfile", path.Join(config.GetString("rundir"), "glusterd2.pid")) } - // Set default peer port. This shouldn't be configurable. - config.SetDefault("defaultpeerport", defaultPeerAddress[1:]) - // Set peer address. host, port, err := net.SplitHostPort(config.GetString("peeraddress")) if err != nil { @@ -110,7 +96,7 @@ func setDefaults() error { if port == "" { port = config.GetString("defaultpeerport") } - config.SetDefault("peeraddress", host+":"+port) + config.Set("peeraddress", host+":"+port) return nil } @@ -143,26 +129,38 @@ func initConfig(confFile string) error { // Limit config to toml only to avoid confusion with multiple config types config.SetConfigType("toml") - - if confFile == "" { - config.SetConfigName(defaultConfName) - for _, p := range defaultConfPaths { - config.AddConfigPath(p) + config.SetConfigName(defaultConfName) + + // Set default config dir and path if default config file exists + // Chances of not having default config file is only during development + // Add current directory to this path if default conf file exists + confdir := defaultConfDir + conffile := defaultConfDir + "/" + defaultConfName + ".toml" + if _, err := os.Stat(defaultConfDir + "/" + defaultConfName + ".toml"); os.IsNotExist(err) { + cdir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return err } - } else { - config.SetConfigFile(confFile) + confdir = cdir + conffile = cdir + "/" + defaultConfName + ".toml" + log.Info("default config file not found, loading config file from current directory") + } + + config.AddConfigPath(confdir) + if err := config.MergeInConfig(); err != nil { + log.WithError(err). + WithField("file", conffile). + Error("failed to read default config file") + return err } - if err := config.ReadInConfig(); err != nil { - if confFile == "" { - log.WithFields(log.Fields{ - "paths": defaultConfPaths, - "config": defaultConfName + ".toml", - "error": err, - }).Debug("failed to read any config files, continuing with defaults") - } else { - log.WithError(err).WithField("file", confFile).Error( - "failed to read given config file") + // If custom configuration is passed + if confFile != "" { + config.SetConfigFile(confFile) + if err := config.MergeInConfig(); err != nil { + log.WithError(err). + WithField("file", confFile). + Error("failed to read config file") return err } } diff --git a/scripts/gen-gd2conf.sh b/scripts/gen-gd2conf.sh index 6638aba2c..96f9c25db 100755 --- a/scripts/gen-gd2conf.sh +++ b/scripts/gen-gd2conf.sh @@ -1,10 +1,15 @@ #!/bin/bash PREFIX=${PREFIX:-/usr/local} +BASE_PREFIX=$PREFIX +if [ "$PREFIX" = "/usr" ]; then + BASE_PREFIX="" +fi + DATADIR=${DATADIR:-$PREFIX/share} LOCALSTATEDIR=${LOCALSTATEDIR:-$PREFIX/var/lib} -LOGDIR=${LOGDIR:-$PREFIX/var/log} -RUNDIR=${RUNDIR:-$PREFIX/var/run} +LOGDIR=${LOGDIR:-$BASE_PREFIX/var/log} +RUNDIR=${RUNDIR:-$BASE_PREFIX/var/run} GD2="glusterd2" GD2STATEDIR=${GD2STATEDIR:-$LOCALSTATEDIR/$GD2} @@ -18,8 +23,14 @@ OUTPUT=$OUTDIR/$GD2.toml cat >"$OUTPUT" <"$OUTPUT" < Date: Tue, 22 May 2018 10:20:21 +0530 Subject: [PATCH 18/54] service health check URL for glusterd2 consumers wanted to to check the status of glusterd2. they can consume this URL to check the status of glusterd2. Signed-off-by: Madhu Rajanna --- doc/endpoints.md | 1 + glusterd2/servers/rest/rest.go | 8 ++++++++ glusterd2/servers/rest/routes.go | 5 +++++ pkg/api/volume_resp.go | 4 ++-- pkg/restclient/common.go | 5 +++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/doc/endpoints.md b/doc/endpoints.md index d4427f1b7..19a9573b2 100644 --- a/doc/endpoints.md +++ b/doc/endpoints.md @@ -72,3 +72,4 @@ GlustershDisable | POST | /volumes/{name}/heal/disable | [](https://godoc.org/gi DeviceAdd | POST | /peers/{peerid}/devices | [AddDeviceReq](https://godoc.org/github.com/gluster/glusterd2/pkg/api#AddDeviceReq) | [AddDeviceResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#AddDeviceResp) Statedump | GET | /statedump | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) List Endpoints | GET | /endpoints | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [ListEndpointsResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#ListEndpointsResp) +Glusterd2 service status | GET | /ping | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) diff --git a/glusterd2/servers/rest/rest.go b/glusterd2/servers/rest/rest.go index 8d093d528..13a55f144 100644 --- a/glusterd2/servers/rest/rest.go +++ b/glusterd2/servers/rest/rest.go @@ -135,3 +135,11 @@ func (r *GDRest) listEndpointsHandler() http.HandlerFunc { restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) }) } + +//Ping URL for glusterd2 +func (r *GDRest) Ping() http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + restutils.SendHTTPResponse(ctx, w, http.StatusOK, nil) + }) +} diff --git a/glusterd2/servers/rest/routes.go b/glusterd2/servers/rest/routes.go index 0fc5839d3..23e1b74ab 100644 --- a/glusterd2/servers/rest/routes.go +++ b/glusterd2/servers/rest/routes.go @@ -80,5 +80,10 @@ func (r *GDRest) registerRoutes() { ResponseType: utils.GetTypeString((*api.ListEndpointsResp)(nil)), HandlerFunc: r.listEndpointsHandler()}) + moreRoutes = append(moreRoutes, route.Route{ + Name: "Glusterd2 service status", + Method: "GET", + Pattern: "/ping", + HandlerFunc: r.Ping()}) r.setRoutes(moreRoutes) } diff --git a/pkg/api/volume_resp.go b/pkg/api/volume_resp.go index b148ed202..bf7ff4bdc 100644 --- a/pkg/api/volume_resp.go +++ b/pkg/api/volume_resp.go @@ -85,7 +85,7 @@ Example of API request- b- GET 'http://localhost:24007/v1/volumes?key={keyname}' c- GET 'http://localhost:24007/v1/volumes?value={value}' Note - Cannot use query parameters or cli flags if volname is also supplied. -*/ +*/ type VolumeGetResp VolumeInfo // VolumeExpandResp is the response sent for a volume expand request. @@ -110,7 +110,7 @@ Example of API request- b- GET 'http://localhost:24007/v1/volumes?key={keyname}' c- GET 'http://localhost:24007/v1/volumes?value={value}' Note - Cannot use query parameters or cli flags if volname is also supplied. -*/ +*/ type VolumeListResp []VolumeGetResp // OptionGroupListResp is the response sent for a group list request. diff --git a/pkg/restclient/common.go b/pkg/restclient/common.go index 0a368bf79..3e97792e2 100644 --- a/pkg/restclient/common.go +++ b/pkg/restclient/common.go @@ -145,3 +145,8 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod return nil } + +//Ping checks glusterd2 service status +func (c *Client) Ping() error { + return c.get("/ping", nil, http.StatusOK, nil) +} From 55ed87fc5377d80eae994ec9f7ffa152a94e9fc6 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Thu, 17 May 2018 16:02:12 +0530 Subject: [PATCH 19/54] added requestbody for volume start api Signed-off-by: Madhu Rajanna --- doc/endpoints.md | 2 +- e2e/glustershd_test.go | 2 +- e2e/quota_enable.go | 2 +- e2e/volume_ops_test.go | 5 ++--- glustercli/cmd/volume.go | 2 +- glusterd2/commands/volumes/commands.go | 1 + glusterd2/commands/volumes/volume-start.go | 11 ++++++++++- pkg/api/volume_req.go | 5 +++++ pkg/api/volume_resp.go | 4 ++-- pkg/restclient/volume.go | 7 +++++-- 10 files changed, 29 insertions(+), 12 deletions(-) diff --git a/doc/endpoints.md b/doc/endpoints.md index d4427f1b7..5da4a33d1 100644 --- a/doc/endpoints.md +++ b/doc/endpoints.md @@ -26,7 +26,7 @@ VolumeInfo | GET | /volumes/{volname} | [](https://godoc.org/github.com/gluster/ VolumeBricksStatus | GET | /volumes/{volname}/bricks | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [BricksStatusResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#BricksStatusResp) VolumeStatus | GET | /volumes/{volname}/status | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [VolumeStatusResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeStatusResp) VolumeList | GET | /volumes | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [VolumeListResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeListResp) -VolumeStart | POST | /volumes/{volname}/start | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [VolumeStartResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeStartResp) +VolumeStart | POST | /volumes/{volname}/start | [VolumeStartReq](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeStartReq) | [VolumeStartResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeStartResp) VolumeStop | POST | /volumes/{volname}/stop | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [VolumeStopResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeStopResp) Statedump | POST | /volumes/{volname}/statedump | [VolStatedumpReq](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolStatedumpReq) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) VolfilesGenerate | POST | /volfiles | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) diff --git a/e2e/glustershd_test.go b/e2e/glustershd_test.go index 515b66d7d..9ca1a19a4 100644 --- a/e2e/glustershd_test.go +++ b/e2e/glustershd_test.go @@ -47,7 +47,7 @@ func TestGlusterShd(t *testing.T) { vol1, err := client.VolumeCreate(reqVol) r.Nil(err) - r.Nil(client.VolumeStart(vol1.Name), "volume start failed") + r.Nil(client.VolumeStart(vol1.Name, false), "volume start failed") err = client.GlusterShdEnable(vol1.Name) r.Nil(err) diff --git a/e2e/quota_enable.go b/e2e/quota_enable.go index 376321b1d..659d5650c 100644 --- a/e2e/quota_enable.go +++ b/e2e/quota_enable.go @@ -50,7 +50,7 @@ func testQuotaEnable(t *testing.T) { vol1, err := client.VolumeCreate(reqVol) r.Nil(err) - r.Nil(client.VolumeStart(vol1.Name), "volume start failed") + r.Nil(client.VolumeStart(vol1.Name, false), "volume start failed") err = client.QuotaEnable(volname) r.Nil(err) diff --git a/e2e/volume_ops_test.go b/e2e/volume_ops_test.go index 087586202..469a0aa27 100644 --- a/e2e/volume_ops_test.go +++ b/e2e/volume_ops_test.go @@ -144,8 +144,7 @@ func testVolumeDelete(t *testing.T) { func testVolumeStart(t *testing.T) { r := require.New(t) - - r.Nil(client.VolumeStart(volname), "volume start failed") + r.Nil(client.VolumeStart(volname, false), "volume start failed") } func testVolumeStop(t *testing.T) { @@ -433,7 +432,7 @@ func testDisperse(t *testing.T) { _, err := client.VolumeCreate(createReq) r.Nil(err) - r.Nil(client.VolumeStart(disperseVolName), "disperse volume start failed") + r.Nil(client.VolumeStart(disperseVolName, true), "disperse volume start failed") mntPath, err := ioutil.TempDir(tmpDir, "mnt") r.Nil(err) diff --git a/glustercli/cmd/volume.go b/glustercli/cmd/volume.go index 5be764940..6a1c43de1 100644 --- a/glustercli/cmd/volume.go +++ b/glustercli/cmd/volume.go @@ -165,7 +165,7 @@ var volumeStartCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { volname := cmd.Flags().Args()[0] - err := client.VolumeStart(volname) + err := client.VolumeStart(volname, flagStartCmdForce) if err != nil { if verbose { log.WithFields(log.Fields{ diff --git a/glusterd2/commands/volumes/commands.go b/glusterd2/commands/volumes/commands.go index c8359b636..dcafd986c 100644 --- a/glusterd2/commands/volumes/commands.go +++ b/glusterd2/commands/volumes/commands.go @@ -106,6 +106,7 @@ func (c *Command) Routes() route.Routes { Method: "POST", Pattern: "/volumes/{volname}/start", Version: 1, + RequestType: utils.GetTypeString((*api.VolumeStartReq)(nil)), ResponseType: utils.GetTypeString((*api.VolumeStartResp)(nil)), HandlerFunc: volumeStartHandler}, route.Route{ diff --git a/glusterd2/commands/volumes/volume-start.go b/glusterd2/commands/volumes/volume-start.go index 077948b77..34658fe50 100644 --- a/glusterd2/commands/volumes/volume-start.go +++ b/glusterd2/commands/volumes/volume-start.go @@ -30,6 +30,9 @@ func startAllBricks(c transaction.TxnCtx) error { }).Info("Starting brick") if err := b.StartBrick(); err != nil { + if err == errors.ErrProcessAlreadyRunning { + continue + } return err } } @@ -68,6 +71,12 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) volname := mux.Vars(r)["volname"] + var req api.VolumeStartReq + + if err := restutils.UnmarshalRequest(r, &req); err != nil { + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) + return + } lock, unlock := transaction.CreateLockFuncs(volname) // Taking a lock outside the txn as volinfo.Nodes() must also @@ -92,7 +101,7 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { return } - if volinfo.State == volume.VolStarted { + if volinfo.State == volume.VolStarted && !req.ForceStartBricks { restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrVolAlreadyStarted) return } diff --git a/pkg/api/volume_req.go b/pkg/api/volume_req.go index d667e9e64..d1fa8e946 100644 --- a/pkg/api/volume_req.go +++ b/pkg/api/volume_req.go @@ -94,3 +94,8 @@ type VolEditReq struct { Metadata map[string]string `json:"metadata"` DeleteMetadata bool `json:"delete-metadata"` } + +// VolumeStartReq represents a request to start volume +type VolumeStartReq struct { + ForceStartBricks bool `json:"force-start-bricks,omitempty"` +} diff --git a/pkg/api/volume_resp.go b/pkg/api/volume_resp.go index b148ed202..bf7ff4bdc 100644 --- a/pkg/api/volume_resp.go +++ b/pkg/api/volume_resp.go @@ -85,7 +85,7 @@ Example of API request- b- GET 'http://localhost:24007/v1/volumes?key={keyname}' c- GET 'http://localhost:24007/v1/volumes?value={value}' Note - Cannot use query parameters or cli flags if volname is also supplied. -*/ +*/ type VolumeGetResp VolumeInfo // VolumeExpandResp is the response sent for a volume expand request. @@ -110,7 +110,7 @@ Example of API request- b- GET 'http://localhost:24007/v1/volumes?key={keyname}' c- GET 'http://localhost:24007/v1/volumes?value={value}' Note - Cannot use query parameters or cli flags if volname is also supplied. -*/ +*/ type VolumeListResp []VolumeGetResp // OptionGroupListResp is the response sent for a group list request. diff --git a/pkg/restclient/volume.go b/pkg/restclient/volume.go index 8b3b449dd..f581e2c1c 100644 --- a/pkg/restclient/volume.go +++ b/pkg/restclient/volume.go @@ -90,9 +90,12 @@ func (c *Client) VolumeStatus(volname string) (api.VolumeStatusResp, error) { } // VolumeStart starts a Gluster Volume -func (c *Client) VolumeStart(volname string) error { +func (c *Client) VolumeStart(volname string, force bool) error { + req := api.VolumeStartReq{ + ForceStartBricks: force, + } url := fmt.Sprintf("/v1/volumes/%s/start", volname) - return c.post(url, nil, http.StatusOK, nil) + return c.post(url, req, http.StatusOK, nil) } // VolumeStop stops a Gluster Volume From 30390198d48105297f8dd23f2ab3df532ef0389c Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 22 May 2018 16:00:34 -0400 Subject: [PATCH 20/54] makefile: support generating sources via a standalone target I tried to run 'make test' out of a clean repo and it failed due to missing source files that were being generated as part of the build. I have tweaked the Makefile so that a new target 'generated' is a prerequisite for the tests and build. In the future we might consider switching to calling something like 'go generate' here. But I'm trying to keep this change small. Signed-off-by: John Mulligan --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a9a27d8a7..ef7612e72 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ DEPENV ?= PLUGINS ?= yes FASTBUILD ?= yes -.PHONY: all build check check-go check-reqs install vendor-update vendor-install verify release check-protoc $(GD2_BIN) $(GD2_BUILD) $(CLI_BIN) $(CLI_BUILD) cli $(GD2_CONF) gd2conf test dist dist-vendor functest +.PHONY: all build check check-go check-reqs install vendor-update vendor-install verify release check-protoc $(GD2_BIN) $(GD2_BUILD) $(CLI_BIN) $(CLI_BUILD) cli $(GD2_CONF) gd2conf test dist dist-vendor functest generated all: build @@ -49,12 +49,16 @@ check-reqs: @./scripts/check-reqs.sh @echo -$(GD2_BIN): $(GD2_BUILD) gd2conf -$(GD2_BUILD): +generated: + @echo "Generating sources..." @PREFIX=$(PREFIX) BASE_PREFIX=$(BASE_PREFIX) EXEC_PREFIX=$(EXEC_PREFIX) \ BINDIR=$(BINDIR) SBINDIR=$(SBINDIR) DATADIR=$(DATADIR) \ LOCALSTATEDIR=$(LOCALSTATEDIR) LOGDIR=$(LOGDIR) \ SYSCONFDIR=$(SYSCONFDIR) ./scripts/prepare_path_config.sh glusterd2 + + +$(GD2_BIN): $(GD2_BUILD) gd2conf generated +$(GD2_BUILD): generated @PLUGINS=$(PLUGINS) FASTBUILD=$(FASTBUILD) ./scripts/build.sh glusterd2 @echo @@ -86,10 +90,10 @@ vendor-install: @$(DEPENV) dep ensure @echo -test: check-reqs +test: check-reqs generated @./test.sh $(TESTOPTIONS) -functest: check-reqs +functest: check-reqs generated @go test ./e2e -v -functest release: build From 3c9cf5d9e54526f963d08ef2d454401e69c027c0 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 22 May 2018 17:22:03 -0400 Subject: [PATCH 21/54] convert daemon interface to use vector args instead of string This change converts the Daemon interface to use an Args function that returns a []string rather than a string. It's generally good policy to keep your argv as a distinct set of arguments rather than mush everything together in a string that must be pulled apart again later. It's easier to see where each argument begins and ends, it works better for (and futureproofs against) arguments that may contain spaces, and can potentially avoid security issues if some of the input strings contain unusual sequences of characters, and it even ends up needing less typing. I had some questions around the storeddaemon and events changes. I'd like to hear if anyone has opinions about that part of the change. Signed-off-by: John Mulligan --- glusterd2/brick/glusterfsd.go | 34 +++++++++++++++++--------------- glusterd2/daemon/daemon.go | 7 +++---- glusterd2/daemon/events.go | 3 ++- glusterd2/daemon/storeddaemon.go | 6 ++++-- plugins/bitrot/bitd.go | 20 +++++++++---------- plugins/bitrot/scrubd.go | 20 +++++++++---------- plugins/georeplication/gsyncd.go | 24 +++++++++++----------- plugins/glustershd/glustershd.go | 22 ++++++++++----------- plugins/quota/quotad.go | 28 ++++++++++++-------------- 9 files changed, 81 insertions(+), 83 deletions(-) diff --git a/glusterd2/brick/glusterfsd.go b/glusterd2/brick/glusterfsd.go index 73c958026..239ecc49a 100644 --- a/glusterd2/brick/glusterfsd.go +++ b/glusterd2/brick/glusterfsd.go @@ -1,7 +1,6 @@ package brick import ( - "bytes" "fmt" "net" "os/exec" @@ -29,7 +28,7 @@ const ( type Glusterfsd struct { // Externally consumable using methods of Glusterfsd interface binarypath string - args string + args []string socketfilepath string pidfilepath string @@ -48,7 +47,7 @@ func (b *Glusterfsd) Path() string { } // Args returns arguments to be passed to brick process during spawn. -func (b *Glusterfsd) Args() string { +func (b *Glusterfsd) Args() []string { brickPathWithoutSlashes := strings.Trim(strings.Replace(b.brickinfo.Path, "/", "-", -1), "-") @@ -63,19 +62,22 @@ func (b *Glusterfsd) Args() string { shost = "127.0.0.1" } - var buffer bytes.Buffer - buffer.WriteString(fmt.Sprintf(" --volfile-server %s", shost)) - buffer.WriteString(fmt.Sprintf(" --volfile-server-port %s", sport)) - buffer.WriteString(fmt.Sprintf(" --volfile-id %s", volFileID)) - buffer.WriteString(fmt.Sprintf(" -p %s", b.PidFile())) - buffer.WriteString(fmt.Sprintf(" -S %s", b.SocketFile())) - buffer.WriteString(fmt.Sprintf(" --brick-name %s", b.brickinfo.Path)) - buffer.WriteString(fmt.Sprintf(" --brick-port %s", brickPort)) - buffer.WriteString(fmt.Sprintf(" -l %s", logFile)) - buffer.WriteString(fmt.Sprintf(" --xlator-option *-posix.glusterd-uuid=%s", gdctx.MyUUID)) - buffer.WriteString(fmt.Sprintf(" --xlator-option %s-server.transport.socket.listen-port=%s", b.brickinfo.VolumeName, brickPort)) - - b.args = buffer.String() + b.args = []string{} + b.args = append(b.args, "--volfile-server", shost) + b.args = append(b.args, "--volfile-server-port", sport) + b.args = append(b.args, "--volfile-id", volFileID) + b.args = append(b.args, "-p", b.PidFile()) + b.args = append(b.args, "-S", b.SocketFile()) + b.args = append(b.args, "--brick-name", b.brickinfo.Path) + b.args = append(b.args, "--brick-port", brickPort) + b.args = append(b.args, "-l", logFile) + b.args = append(b.args, + "--xlator-option", + fmt.Sprintf("*-posix.glusterd-uuid=%s", gdctx.MyUUID)) + b.args = append(b.args, + "--xlator-option", + fmt.Sprintf("%s-server.transport.socket.listen-port=%s", b.brickinfo.VolumeName, brickPort)) + return b.args } diff --git a/glusterd2/daemon/daemon.go b/glusterd2/daemon/daemon.go index e7c97efae..06fc3d676 100644 --- a/glusterd2/daemon/daemon.go +++ b/glusterd2/daemon/daemon.go @@ -26,7 +26,7 @@ type Daemon interface { // Args should return the arguments to be passed to the binary // during spawn. - Args() string + Args() []string // SocketFile should return the absolute path to the socket file which // will be used for inter process communication using Unix domain @@ -56,7 +56,7 @@ func Start(d Daemon, wait bool) error { log.WithFields(log.Fields{ "name": d.Name(), "path": d.Path(), - "args": d.Args(), + "args": strings.Join(d.Args(), " "), }).Debug("Starting daemon.") events.Broadcast(newEvent(d, daemonStarting, 0)) @@ -71,8 +71,7 @@ func Start(d Daemon, wait bool) error { } } - args := strings.Fields(d.Args()) - cmd := exec.Command(d.Path(), args...) + cmd := exec.Command(d.Path(), d.Args()...) err = cmd.Start() if err != nil { events.Broadcast(newEvent(d, daemonStartFailed, 0)) diff --git a/glusterd2/daemon/events.go b/glusterd2/daemon/events.go index ac3001b09..e34029a3d 100644 --- a/glusterd2/daemon/events.go +++ b/glusterd2/daemon/events.go @@ -2,6 +2,7 @@ package daemon import ( "strconv" + "strings" "github.com/gluster/glusterd2/glusterd2/events" "github.com/gluster/glusterd2/pkg/api" @@ -27,7 +28,7 @@ func newEvent(d Daemon, e daemonEvent, pid int) *api.Event { "name": d.Name(), "id": d.ID(), "binary": d.Path(), - "args": d.Args(), + "args": strings.Join(d.Args(), " "), "pid": strconv.Itoa(pid), } diff --git a/glusterd2/daemon/storeddaemon.go b/glusterd2/daemon/storeddaemon.go index 6d90a4f56..4ea7eea5d 100644 --- a/glusterd2/daemon/storeddaemon.go +++ b/glusterd2/daemon/storeddaemon.go @@ -3,7 +3,9 @@ package daemon // storedDaemon is used to save/retrieve a daemons information in the store, // and also implements the Daemon interface type storedDaemon struct { - DName, DPath, DArgs, DSocketFile, DPidFile, DID string + DName, DPath, DSocketFile, DPidFile, DID string + + DArgs []string } func newStoredDaemon(d Daemon) *storedDaemon { @@ -25,7 +27,7 @@ func (s *storedDaemon) Path() string { return s.DPath } -func (s *storedDaemon) Args() string { +func (s *storedDaemon) Args() []string { return s.DArgs } diff --git a/plugins/bitrot/bitd.go b/plugins/bitrot/bitd.go index 89a2f7496..4f52c1940 100644 --- a/plugins/bitrot/bitd.go +++ b/plugins/bitrot/bitd.go @@ -1,7 +1,6 @@ package bitrot import ( - "bytes" "fmt" "net" "os" @@ -19,7 +18,7 @@ const ( // Bitd type represents information about bitd process type Bitd struct { - args string + args []string pidfilepath string binarypath string volfileID string @@ -38,7 +37,7 @@ func (b *Bitd) Path() string { } // Args returns arguments to be passed to bitd process during spawn. -func (b *Bitd) Args() string { +func (b *Bitd) Args() []string { return b.args } @@ -84,14 +83,13 @@ func newBitd() (*Bitd, error) { shost = "localhost" } - var buffer bytes.Buffer - buffer.WriteString(fmt.Sprintf(" -s %s", shost)) - buffer.WriteString(fmt.Sprintf(" --volfile-id %s", b.volfileID)) - buffer.WriteString(fmt.Sprintf(" -p %s", b.pidfilepath)) - buffer.WriteString(fmt.Sprintf(" -l %s", b.logfile)) - buffer.WriteString(fmt.Sprintf(" -S %s", b.socketfilepath)) - buffer.WriteString(fmt.Sprintf(" --global-timer-wheel")) - b.args = buffer.String() + b.args = []string{} + b.args = append(b.args, "-s", shost) + b.args = append(b.args, "--volfile-id", b.volfileID) + b.args = append(b.args, "-p", b.pidfilepath) + b.args = append(b.args, "-l", b.logfile) + b.args = append(b.args, "-S", b.socketfilepath) + b.args = append(b.args, "--global-timer-wheel") return b, nil } diff --git a/plugins/bitrot/scrubd.go b/plugins/bitrot/scrubd.go index 7927345b4..aa606485f 100644 --- a/plugins/bitrot/scrubd.go +++ b/plugins/bitrot/scrubd.go @@ -1,7 +1,6 @@ package bitrot import ( - "bytes" "fmt" "net" "os" @@ -19,7 +18,7 @@ const ( // Scrubd type represents information about scrubd process type Scrubd struct { - args string + args []string pidfilepath string binarypath string volfileID string @@ -38,7 +37,7 @@ func (s *Scrubd) Path() string { } // Args returns arguments to be passed to scrubd process during spawn. -func (s *Scrubd) Args() string { +func (s *Scrubd) Args() []string { return s.args } @@ -84,14 +83,13 @@ func newScrubd() (*Scrubd, error) { shost = "localhost" } - var buffer bytes.Buffer - buffer.WriteString(fmt.Sprintf(" -s %s", shost)) - buffer.WriteString(fmt.Sprintf(" --volfile-id %s", s.volfileID)) - buffer.WriteString(fmt.Sprintf(" -p %s", s.pidfilepath)) - buffer.WriteString(fmt.Sprintf(" -l %s", s.logfile)) - buffer.WriteString(fmt.Sprintf(" -S %s", s.socketfilepath)) - buffer.WriteString(fmt.Sprintf(" --global-timer-wheel")) - s.args = buffer.String() + s.args = []string{} + s.args = append(s.args, "-s", shost) + s.args = append(s.args, "--volfile-id", s.volfileID) + s.args = append(s.args, "-p", s.pidfilepath) + s.args = append(s.args, "-l", s.logfile) + s.args = append(s.args, "-S", s.socketfilepath) + s.args = append(s.args, "--global-timer-wheel") return s, nil } diff --git a/plugins/georeplication/gsyncd.go b/plugins/georeplication/gsyncd.go index 04883464a..ae281060b 100644 --- a/plugins/georeplication/gsyncd.go +++ b/plugins/georeplication/gsyncd.go @@ -1,7 +1,6 @@ package georeplication import ( - "bytes" "fmt" "path" @@ -19,7 +18,7 @@ const ( type Gsyncd struct { // Externally consumable using methods of Gsyncd interface binarypath string - args string + args []string configFilePath string pidfilepath string // For internal use @@ -37,16 +36,17 @@ func (g *Gsyncd) Path() string { } // Args returns arguments to be passed to gsyncd process during spawn. -func (g *Gsyncd) Args() string { - var buffer bytes.Buffer - buffer.WriteString(" monitor") - buffer.WriteString(fmt.Sprintf(" %s", g.sessioninfo.MasterVol)) - buffer.WriteString(fmt.Sprintf(" %s@%s::%s", g.sessioninfo.RemoteUser, g.sessioninfo.RemoteHosts[0].Hostname, g.sessioninfo.RemoteVol)) - buffer.WriteString(fmt.Sprintf(" --local-node-id %s", gdctx.MyUUID.String())) - buffer.WriteString(fmt.Sprintf(" -c %s", g.ConfigFile())) - buffer.WriteString(" --use-gconf-volinfo") - - g.args = buffer.String() +func (g *Gsyncd) Args() []string { + g.args = []string{} + + g.args = append(g.args, "monitor") + g.args = append(g.args, g.sessioninfo.MasterVol) + g.args = append(g.args, + fmt.Sprintf("%s@%s::%s", g.sessioninfo.RemoteUser, g.sessioninfo.RemoteHosts[0].Hostname, g.sessioninfo.RemoteVol)) + g.args = append(g.args, "--local-node-id", gdctx.MyUUID.String()) + g.args = append(g.args, "-c", g.ConfigFile()) + g.args = append(g.args, "--use-gconf-volinfo") + return g.args } diff --git a/plugins/glustershd/glustershd.go b/plugins/glustershd/glustershd.go index dbd331a79..18c652a94 100644 --- a/plugins/glustershd/glustershd.go +++ b/plugins/glustershd/glustershd.go @@ -1,7 +1,6 @@ package glustershd import ( - "bytes" "fmt" "net" "os/exec" @@ -20,7 +19,7 @@ const ( // Glustershd type represents information about Glustershd process type Glustershd struct { binarypath string - args string + args []string socketfilepath string pidfilepath string } @@ -37,7 +36,7 @@ func (shd *Glustershd) Path() string { } // Args returns arguments to be passed to glustershd process during spawn. -func (shd *Glustershd) Args() string { +func (shd *Glustershd) Args() []string { shost, _, _ := net.SplitHostPort(config.GetString("clientaddress")) if shost == "" { shost = "localhost" @@ -48,15 +47,16 @@ func (shd *Glustershd) Args() string { glusterdSockDir := config.GetString("rundir") socketfilepath := fmt.Sprintf("%s/%x.socket", glusterdSockDir, xxhash.Sum64String(gdctx.MyUUID.String())) - var buffer bytes.Buffer - buffer.WriteString(fmt.Sprintf(" -s %s", shost)) - buffer.WriteString(fmt.Sprintf(" --volfile-id %s", volFileID)) - buffer.WriteString(fmt.Sprintf(" -p %s", shd.PidFile())) - buffer.WriteString(fmt.Sprintf(" -l %s", logFile)) - buffer.WriteString(fmt.Sprintf(" -S %s", socketfilepath)) - buffer.WriteString(fmt.Sprintf(" --xlator-option *replicate*.node-uuid=%s", gdctx.MyUUID)) + shd.args = []string{} + shd.args = append(shd.args, "-s", shost) + shd.args = append(shd.args, "--volfile-id", volFileID) + shd.args = append(shd.args, "-p", shd.PidFile()) + shd.args = append(shd.args, "-l", logFile) + shd.args = append(shd.args, "-S", socketfilepath) + shd.args = append(shd.args, + "--xlator-option", + fmt.Sprintf("*replicate*.node-uuid=%s", gdctx.MyUUID)) - shd.args = buffer.String() return shd.args } diff --git a/plugins/quota/quotad.go b/plugins/quota/quotad.go index 71e042624..15cbef191 100644 --- a/plugins/quota/quotad.go +++ b/plugins/quota/quotad.go @@ -1,7 +1,6 @@ package quota import ( - "bytes" "fmt" "net" "os/exec" @@ -20,7 +19,7 @@ const ( type Quotad struct { // Externally consumable using methods of Quotad interface binarypath string - args string + args []string socketfilepath string pidfilepath string logfilepath string @@ -41,7 +40,7 @@ func (q *Quotad) Path() string { } // Args returns arguments to be passed to quota process during spawn. -func (q *Quotad) Args() string { +func (q *Quotad) Args() []string { shost, _, _ := net.SplitHostPort(config.GetString("clientaddress")) if shost == "" { @@ -49,18 +48,17 @@ func (q *Quotad) Args() string { } q.volFileID = "gluster/quotad" - var buffer bytes.Buffer - buffer.WriteString(fmt.Sprintf(" -s %s", shost)) - buffer.WriteString(fmt.Sprintf(" --volfile-id %s", q.volFileID)) - buffer.WriteString(fmt.Sprintf(" -p %s", q.PidFile())) - buffer.WriteString(fmt.Sprintf(" -S %s", q.SocketFile())) - buffer.WriteString(fmt.Sprintf(" -l %s", q.logfilepath)) - buffer.WriteString(fmt.Sprintf(" --process-name quotad")) - buffer.WriteString(fmt.Sprintf(" --xlator-option *replicate*.entry-self-heal=off")) - buffer.WriteString(fmt.Sprintf(" --xlator-option *replicate*.metadata-self-heal=off")) - buffer.WriteString(fmt.Sprintf(" --xlator-option *replicate*.data-self-heal=off")) - - q.args = buffer.String() + q.args = []string{} + q.args = append(q.args, "-s", shost) + q.args = append(q.args, "--volfile-id", q.volFileID) + q.args = append(q.args, "-p", q.PidFile()) + q.args = append(q.args, "-S", q.SocketFile()) + q.args = append(q.args, "-l", q.logfilepath) + q.args = append(q.args, "--process-name", "quotad") + q.args = append(q.args, "--xlator-option", "*replicate*.entry-self-heal=off") + q.args = append(q.args, "--xlator-option", "*replicate*.metadata-self-heal=off") + q.args = append(q.args, "--xlator-option", "replicate*.data-self-heal=off") + return q.args } From 9aff829261fdf261e4c0bd0d9ae929d12f73ca5c Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Wed, 23 May 2018 01:43:27 -0400 Subject: [PATCH 22/54] List devices --- plugins/device/api/resp.go | 3 +++ plugins/device/init.go | 7 +++++++ plugins/device/rest.go | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index fccda3874..ce085e249 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -12,3 +12,6 @@ type Info struct { // AddDeviceResp is the success response sent to a AddDeviceReq request type AddDeviceResp api.Peer + +// ListDeviceResp is the success response sent to a ListDevice request +type ListDeviceResp []Info diff --git a/plugins/device/init.go b/plugins/device/init.go index 8233ab97e..b960c226f 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -27,6 +27,13 @@ func (p *Plugin) RestRoutes() route.Routes { RequestType: utils.GetTypeString((*deviceapi.AddDeviceReq)(nil)), ResponseType: utils.GetTypeString((*deviceapi.AddDeviceResp)(nil)), HandlerFunc: deviceAddHandler}, + route.Route{ + Name: "DeviceList", + Method: "GET", + Pattern: "/peers/{peerid}/devices", + Version: 1, + ResponseType: utils.GetTypeString((*deviceapi.ListDeviceResp)(nil)), + HandlerFunc: deviceListHandler}, } } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index d07a2f524..7e604c5de 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -104,3 +104,28 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { } restutils.SendHTTPResponse(ctx, w, http.StatusOK, peerInfo) } + +func deviceListHandler(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + logger := gdctx.GetReqLogger(ctx) + peerID := mux.Vars(r)["peerid"] + if uuid.Parse(peerID) == nil { + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Invalid peer-id passed in url") + return + } + + devices, err := GetDevices(peerID) + if err != nil { + logger.WithError(err).WithField("peerid", peerID).Error(err) + if err == errors.ErrPeerNotFound { + restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + + restutils.SendHTTPResponse(ctx, w, http.StatusOK, devices) + +} From 8cdab77b23362bf777edf1f6304a28b8ca7a11d2 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Thu, 17 May 2018 13:57:35 +0530 Subject: [PATCH 23/54] store peer address provided during peer add in ETCD. if peer address already present in store update configuration on startup this allows the users to use peer address provided during peer add, for any other operations(ex:- volume create,expand) Signed-off-by: Madhu Rajanna --- glusterd2/commands/peers/addpeer.go | 13 +++++++++++++ glusterd2/peer/self.go | 11 +++++++++++ pkg/utils/peerutils.go | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/glusterd2/commands/peers/addpeer.go b/glusterd2/commands/peers/addpeer.go index 381dfef81..5b536b2ce 100644 --- a/glusterd2/commands/peers/addpeer.go +++ b/glusterd2/commands/peers/addpeer.go @@ -3,6 +3,7 @@ package peercommands import ( "fmt" "net/http" + "sort" "strings" "github.com/gluster/glusterd2/glusterd2/events" @@ -99,10 +100,22 @@ func addPeerHandler(w http.ResponseWriter, r *http.Request) { newpeer.Metadata[key] = value } + //check if remotePeerAddress already present + found := utils.StringInSlice(remotePeerAddress, newpeer.PeerAddresses) + //if not found prepend the remotePeerAddress to peer details + if !found { + newpeer.PeerAddresses = append([]string{remotePeerAddress}, newpeer.PeerAddresses...) + } else { + index := sort.StringSlice(newpeer.PeerAddresses).Search(remotePeerAddress) + //swap peer address to index 0 + sort.StringSlice(newpeer.PeerAddresses).Swap(0, index) + } + err = peer.AddOrUpdatePeer(newpeer) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Fail to add metadata to peer") } + resp := createPeerAddResp(newpeer) restutils.SendHTTPResponse(ctx, w, http.StatusCreated, resp) diff --git a/glusterd2/peer/self.go b/glusterd2/peer/self.go index a0c823774..e0a369cbb 100644 --- a/glusterd2/peer/self.go +++ b/glusterd2/peer/self.go @@ -6,6 +6,7 @@ import ( "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/pkg/errors" + "github.com/gluster/glusterd2/pkg/utils" config "github.com/spf13/viper" ) @@ -47,6 +48,7 @@ func AddSelfDetails() error { Name: gdctx.HostName, PeerAddresses: []string{config.GetString("peeraddress")}, } + p.ClientAddresses, err = normalizeAddrs() if err != nil { return err @@ -56,8 +58,17 @@ func AddSelfDetails() error { if err == errors.ErrPeerNotFound { p.Metadata = make(map[string]string) p.Metadata["_zone"] = p.ID.String() + } else if err == nil && peerInfo != nil { p.Metadata = peerInfo.Metadata + + found := utils.StringInSlice(p.PeerAddresses[0], peerInfo.PeerAddresses) + if !found { + p.PeerAddresses = append(peerInfo.PeerAddresses, p.PeerAddresses...) + } else { + p.PeerAddresses = peerInfo.PeerAddresses + } + } else { return err } diff --git a/pkg/utils/peerutils.go b/pkg/utils/peerutils.go index ebeb55b06..7a74d48d1 100644 --- a/pkg/utils/peerutils.go +++ b/pkg/utils/peerutils.go @@ -9,7 +9,7 @@ import ( ) // FormRemotePeerAddress will check and validate peeraddress provided. It will -// return an address of the form +// return an address of the form func FormRemotePeerAddress(peeraddress string) (string, error) { host, port, err := net.SplitHostPort(peeraddress) From 197a143acb98eb8d181cabb8fa3ec4f776d22cd5 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 23 May 2018 11:43:44 +0530 Subject: [PATCH 24/54] resolve documentation issue for volumeGetResp and volumeListResp --- pkg/api/volume_resp.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pkg/api/volume_resp.go b/pkg/api/volume_resp.go index bf7ff4bdc..32ee37d84 100644 --- a/pkg/api/volume_resp.go +++ b/pkg/api/volume_resp.go @@ -76,15 +76,15 @@ type VolumeStatusResp struct { type VolumeCreateResp VolumeInfo // VolumeGetResp is the response sent for a volume get request. -/* VolumeGetResp can also be filtered based on query parameters +/* +VolumeGetResp can also be filtered based on query parameters sent along with volume list/info api. Query Parameters can be either metadata key/value/both key and value. -Example of API request- - 1- /volumes - - a- GET 'http://localhost:24007/v1/volumes?key={keyname}&value={value} - b- GET 'http://localhost:24007/v1/volumes?key={keyname}' - c- GET 'http://localhost:24007/v1/volumes?value={value}' -Note - Cannot use query parameters or cli flags if volname is also supplied. +Example of API request + - GET http://localhost:24007/v1/volumes?key={keyname}&value={value} + - GET http://localhost:24007/v1/volumes?key={keyname} + - GET http://localhost:24007/v1/volumes?value={value} +Note - Cannot use query parameters if volname is also supplied. */ type VolumeGetResp VolumeInfo @@ -101,15 +101,14 @@ type VolumeStopResp VolumeInfo type VolumeOptionResp VolumeInfo // VolumeListResp is the response sent for a volume list request. -/* VolumeListResp can also be filtered based on query parameters +/*VolumeListResp can also be filtered based on query parameters sent along with volume list/info api. Query Parameters can be either metadata key/value/both key and value. -Example of API request- - 1- /volumes - - a- GET 'http://localhost:24007/v1/volumes?key={keyname}&value={value} - b- GET 'http://localhost:24007/v1/volumes?key={keyname}' - c- GET 'http://localhost:24007/v1/volumes?value={value}' -Note - Cannot use query parameters or cli flags if volname is also supplied. +Example of API request + - GET http://localhost:24007/v1/volumes?key={keyname}&value={value} + - GET http://localhost:24007/v1/volumes?key={keyname} + - GET http://localhost:24007/v1/volumes?value={value} +Note - Cannot use query parameters if volname is also supplied. */ type VolumeListResp []VolumeGetResp From d7cd797bc461efa981573714e4f85330dbcfa524 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Wed, 23 May 2018 09:42:21 -0400 Subject: [PATCH 25/54] Updating --- plugins/device/api/resp.go | 7 +++++++ plugins/device/init.go | 11 +++++++++-- plugins/device/rest.go | 26 ++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index ce085e249..4530eb4b9 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -15,3 +15,10 @@ type AddDeviceResp api.Peer // ListDeviceResp is the success response sent to a ListDevice request type ListDeviceResp []Info + +// ListAllDeviceResp is the success response sent to a ListAllDevice request +type ListAllDeviceResp struct { + Name string `json:"name"` + State string `json:"state"` + Peerid string `json:"peerid,omitempty"` +} diff --git a/plugins/device/init.go b/plugins/device/init.go index b960c226f..d58eb0692 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -22,7 +22,7 @@ func (p *Plugin) RestRoutes() route.Routes { route.Route{ Name: "DeviceAdd", Method: "POST", - Pattern: "/peers/{peerid}/devices", + Pattern: "/devices/{peerid}", Version: 1, RequestType: utils.GetTypeString((*deviceapi.AddDeviceReq)(nil)), ResponseType: utils.GetTypeString((*deviceapi.AddDeviceResp)(nil)), @@ -30,10 +30,17 @@ func (p *Plugin) RestRoutes() route.Routes { route.Route{ Name: "DeviceList", Method: "GET", - Pattern: "/peers/{peerid}/devices", + Pattern: "/devices/{peerid}", Version: 1, ResponseType: utils.GetTypeString((*deviceapi.ListDeviceResp)(nil)), HandlerFunc: deviceListHandler}, + route.Route{ + Name: "ListAllDevices", + Method: "GET", + Pattern: "/devices", + Version: 1, + ResponseType: utils.GetTypeString((*[]deviceapi.ListDeviceResp)(nil)), + HandlerFunc: listAllDevicesHandler}, } } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 48e2756fd..f96c768fb 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -129,3 +129,29 @@ func deviceListHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPResponse(ctx, w, http.StatusOK, devices) } + +func listAllHandler(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + logger := gdctx.GetReqLogger(ctx) + peers, err := GetPeers() + if err != nil { + logger.WithError(err).Error("Failed to get peers from store") + } + + for key := range peers { + + devices, err := GetDevices(peerID) + if err != nil { + logger.WithError(err).WithField("peerid", peerID).Error("Failed to get devices for peer") + if err == errors.ErrPeerNotFound { + restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + + restutils.SendHTTPResponse(ctx, w, http.StatusOK, devices) + +} From 649e4d7f0e2f40338811556731dda1543ded777d Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Wed, 23 May 2018 11:56:14 +0530 Subject: [PATCH 26/54] cli: Added CLI for Device Add Also fixed issues during device add(Json Marshal error and store error for first time) Updates: #728 Signed-off-by: Aravinda VK --- glustercli/cmd/device.go | 48 +++++++++++++++++++++++++++++++++++ pkg/restclient/device.go | 17 +++++++++++++ plugins/device/rest.go | 24 ++++++++++-------- plugins/device/store-utils.go | 3 +++ 4 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 glustercli/cmd/device.go create mode 100644 pkg/restclient/device.go diff --git a/glustercli/cmd/device.go b/glustercli/cmd/device.go new file mode 100644 index 000000000..44ec7a3aa --- /dev/null +++ b/glustercli/cmd/device.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +const ( + helpDeviceCmd = "Gluster Devices Management" + helpDeviceAddCmd = "add device" +) + +func init() { + deviceCmd.AddCommand(deviceAddCmd) + + RootCmd.AddCommand(deviceCmd) +} + +var deviceCmd = &cobra.Command{ + Use: "device", + Short: helpDeviceCmd, +} + +var deviceAddCmd = &cobra.Command{ + Use: "add ", + Short: helpDeviceAddCmd, + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + peerid := args[0] + devname := args[1] + + _, err := client.DeviceAdd(peerid, devname) + + if err != nil { + if verbose { + log.WithFields(log.Fields{ + "device": devname, + "peerid": peerid, + "error": err.Error(), + }).Error("device add failed") + } + failure("Device add failed", err, 1) + } + fmt.Println("Device add successful") + }, +} diff --git a/pkg/restclient/device.go b/pkg/restclient/device.go new file mode 100644 index 000000000..9f7020bc5 --- /dev/null +++ b/pkg/restclient/device.go @@ -0,0 +1,17 @@ +package restclient + +import ( + "net/http" + + deviceapi "github.com/gluster/glusterd2/plugins/device/api" +) + +// DeviceAdd registers devices +func (c *Client) DeviceAdd(peerid string, device string) (deviceapi.AddDeviceResp, error) { + var peerinfo deviceapi.AddDeviceResp + req := deviceapi.AddDeviceReq{ + Device: device, + } + err := c.post("/v1/peers/"+peerid+"/devices", req, http.StatusOK, &peerinfo) + return peerinfo, err +} diff --git a/plugins/device/rest.go b/plugins/device/rest.go index d07a2f524..9c7004e58 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -54,17 +54,19 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { return } - var devices []deviceapi.Info - err = json.Unmarshal([]byte(peerInfo.Metadata["_devices"]), &devices) - if err != nil { - logger.WithError(err).WithField("peerid", peerID).Error(err) - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - if checkIfDeviceExist(req.Device, devices) { - logger.WithError(err).WithField("device", req.Device).Error("Device already exists") - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Device already exists") - return + if _, exists := peerInfo.Metadata["_devices"]; exists { + var devices []deviceapi.Info + err = json.Unmarshal([]byte(peerInfo.Metadata["_devices"]), &devices) + if err != nil { + logger.WithError(err).WithField("peerid", peerID).Error(err) + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + return + } + if checkIfDeviceExist(req.Device, devices) { + logger.WithError(err).WithField("device", req.Device).Error("Device already exists") + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Device already exists") + return + } } txn := transaction.NewTxn(ctx) diff --git a/plugins/device/store-utils.go b/plugins/device/store-utils.go index 4ee551c2e..f593f8e99 100644 --- a/plugins/device/store-utils.go +++ b/plugins/device/store-utils.go @@ -44,7 +44,10 @@ func addDevice(device deviceapi.Info, peerID string) error { var devices []deviceapi.Info if deviceDetails != nil { devices = append(deviceDetails, device) + } else { + devices = append(devices, device) } + deviceJSON, err := json.Marshal(devices) if err != nil { return err From 5c0eaeaabe517f22276f2b03c2a1ed77e189f4ae Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Tue, 8 May 2018 16:33:27 +0530 Subject: [PATCH 27/54] Use the built-in "context" package golang.org/x/net/context was moved to golang in go 1.7. NOTE: The rpc files autogenerated by protobuf compiler still imports golang.org/x/net/context Signed-off-by: Prashanth Pai --- glusterd2/commands/peers/peer-rpc-svc.go | 3 ++- glusterd2/transaction/rpc-client.go | 4 ++-- glusterd2/transaction/rpc-service.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/glusterd2/commands/peers/peer-rpc-svc.go b/glusterd2/commands/peers/peer-rpc-svc.go index dc6c03a5b..21ce49739 100644 --- a/glusterd2/commands/peers/peer-rpc-svc.go +++ b/glusterd2/commands/peers/peer-rpc-svc.go @@ -1,6 +1,8 @@ package peercommands import ( + "context" + "github.com/gluster/glusterd2/glusterd2/events" "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/peer" @@ -11,7 +13,6 @@ import ( "github.com/pborman/uuid" log "github.com/sirupsen/logrus" - "golang.org/x/net/context" "google.golang.org/grpc" ) diff --git a/glusterd2/transaction/rpc-client.go b/glusterd2/transaction/rpc-client.go index 5a31f6bc7..8b2548244 100644 --- a/glusterd2/transaction/rpc-client.go +++ b/glusterd2/transaction/rpc-client.go @@ -1,6 +1,7 @@ package transaction import ( + "context" "encoding/json" "errors" @@ -9,7 +10,6 @@ import ( "github.com/pborman/uuid" log "github.com/sirupsen/logrus" - netctx "golang.org/x/net/context" "google.golang.org/grpc" ) @@ -65,7 +65,7 @@ func RunStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { var rsp *TxnStepResp - rsp, err = client.RunStep(netctx.TODO(), req) + rsp, err = client.RunStep(context.TODO(), req) if err != nil { logger.WithFields(log.Fields{ "error": err, diff --git a/glusterd2/transaction/rpc-service.go b/glusterd2/transaction/rpc-service.go index efa8b2fce..b8f4af8a6 100644 --- a/glusterd2/transaction/rpc-service.go +++ b/glusterd2/transaction/rpc-service.go @@ -1,13 +1,13 @@ package transaction import ( + "context" "encoding/json" "errors" "github.com/gluster/glusterd2/glusterd2/servers/peerrpc" log "github.com/sirupsen/logrus" - "golang.org/x/net/context" "google.golang.org/grpc" ) From 48427edd5df478baea92d0d8edb0bb5a346af5dd Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 23 May 2018 22:12:13 +0530 Subject: [PATCH 28/54] txn: Do not export GetStepFunc() and RunStepOn() Signed-off-by: Prashanth Pai --- glusterd2/transaction/lock.go | 4 ++-- glusterd2/transaction/registry.go | 4 ++-- glusterd2/transaction/rpc-client.go | 4 ++-- glusterd2/transaction/rpc-service.go | 2 +- glusterd2/transaction/step.go | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/glusterd2/transaction/lock.go b/glusterd2/transaction/lock.go index 0310ab665..e15e9ccd2 100644 --- a/glusterd2/transaction/lock.go +++ b/glusterd2/transaction/lock.go @@ -27,8 +27,8 @@ func createLockStepFunc(key string) (string, string, error) { lockFuncID := key + ".Lock" unlockFuncID := key + ".Unlock" - _, lockFuncFound := GetStepFunc(lockFuncID) - _, unlockFuncFound := GetStepFunc(unlockFuncID) + _, lockFuncFound := getStepFunc(lockFuncID) + _, unlockFuncFound := getStepFunc(unlockFuncID) if lockFuncFound && unlockFuncFound { return lockFuncID, unlockFuncID, nil diff --git a/glusterd2/transaction/registry.go b/glusterd2/transaction/registry.go index a39c7dd67..e06254a9d 100644 --- a/glusterd2/transaction/registry.go +++ b/glusterd2/transaction/registry.go @@ -33,8 +33,8 @@ func RegisterStepFunc(s StepFunc, name string) { registerStepFunc(s, name) } -//GetStepFunc returns named step if found. -func GetStepFunc(name string) (StepFunc, bool) { +//getStepFunc returns named step if found. +func getStepFunc(name string) (StepFunc, bool) { sfRegistry.RLock() defer sfRegistry.RUnlock() diff --git a/glusterd2/transaction/rpc-client.go b/glusterd2/transaction/rpc-client.go index 8b2548244..3d182f8a6 100644 --- a/glusterd2/transaction/rpc-client.go +++ b/glusterd2/transaction/rpc-client.go @@ -13,8 +13,8 @@ import ( "google.golang.org/grpc" ) -// RunStepOn will run the step on the specified node -func RunStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { +// runStepOn will run the step on the specified node +func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { // TODO: I'm creating connections on demand. This should be changed so that // we have long term connections. p, err := peer.GetPeerF(node.String()) diff --git a/glusterd2/transaction/rpc-service.go b/glusterd2/transaction/rpc-service.go index b8f4af8a6..2af8f596c 100644 --- a/glusterd2/transaction/rpc-service.go +++ b/glusterd2/transaction/rpc-service.go @@ -30,7 +30,7 @@ func (p *txnSvc) RunStep(rpcCtx context.Context, req *TxnStepReq) (*TxnStepResp, logger := ctx.Logger().WithField("stepfunc", req.StepFunc) logger.Debug("RunStep request received") - f, ok := GetStepFunc(req.StepFunc) + f, ok := getStepFunc(req.StepFunc) if !ok { logger.Error("step function not found in registry") return nil, errors.New("step function not found") diff --git a/glusterd2/transaction/step.go b/glusterd2/transaction/step.go index 319fdd3c2..012980106 100644 --- a/glusterd2/transaction/step.go +++ b/glusterd2/transaction/step.go @@ -83,7 +83,7 @@ func runStepFuncOnNode(name string, c TxnCtx, node uuid.UUID, done chan<- error) func runStepFuncLocal(name string, c TxnCtx) error { c.Logger().WithField("stepfunc", name).Debug("running step function") - stepFunc, ok := GetStepFunc(name) + stepFunc, ok := getStepFunc(name) if !ok { return ErrStepFuncNotFound } @@ -92,7 +92,7 @@ func runStepFuncLocal(name string, c TxnCtx) error { } func runStepFuncRemote(step string, c TxnCtx, node uuid.UUID) error { - rsp, err := RunStepOn(step, node, c) + rsp, err := runStepOn(step, node, c) //TODO: Results need to be aggregated _ = rsp return err From 26f13a0b0e96194c3858844f2568b4fdccbb6d37 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 23 May 2018 22:21:42 +0530 Subject: [PATCH 29/54] Remove unused TxnCtx from TxnStepResp Signed-off-by: Prashanth Pai --- glusterd2/transaction/rpc-client.go | 22 ++++------- glusterd2/transaction/rpc-service.go | 8 ---- glusterd2/transaction/step.go | 9 +---- glusterd2/transaction/transaction-rpc.pb.go | 43 ++++++++------------- glusterd2/transaction/transaction-rpc.proto | 1 - 5 files changed, 26 insertions(+), 57 deletions(-) diff --git a/glusterd2/transaction/rpc-client.go b/glusterd2/transaction/rpc-client.go index 3d182f8a6..fcae57c83 100644 --- a/glusterd2/transaction/rpc-client.go +++ b/glusterd2/transaction/rpc-client.go @@ -14,7 +14,7 @@ import ( ) // runStepOn will run the step on the specified node -func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { +func runStepOn(step string, node uuid.UUID, c TxnCtx) error { // TODO: I'm creating connections on demand. This should be changed so that // we have long term connections. p, err := peer.GetPeerF(node.String()) @@ -23,7 +23,7 @@ func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { "peerid": node.String(), "error": err, }).Error("peer not found") - return nil, err + return err } logger := c.Logger().WithField("remotepeer", p.ID.String()+"("+p.Name+")") @@ -32,7 +32,7 @@ func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { remote, err := utils.FormRemotePeerAddress(p.PeerAddresses[0]) if err != nil { - return nil, err + return err } conn, err = grpc.Dial(remote, grpc.WithInsecure()) @@ -47,7 +47,7 @@ func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { "error": err, "remote": p.PeerAddresses[0], }).Error("failed to grpc.Dial remote") - return nil, err + return err } defer conn.Close() @@ -59,7 +59,7 @@ func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { data, err := json.Marshal(c) if err != nil { logger.WithError(err).Error("failed to JSON marshal transaction context") - return nil, err + return err } req.Context = data @@ -71,19 +71,13 @@ func runStepOn(step string, node uuid.UUID, c TxnCtx) (TxnCtx, error) { "error": err, "rpc": "TxnSvc.RunStep", }).Error("failed RPC call") - return nil, err + return err } if rsp.Error != "" { logger.WithError(errors.New(rsp.Error)).Error("TxnSvc.Runstep failed on peer") - return nil, errors.New(rsp.Error) + return errors.New(rsp.Error) } - rspCtx := new(Tctx) - err = json.Unmarshal(rsp.Resp, rspCtx) - if err != nil { - logger.WithError(err).Error("failed to JSON unmarhsal transaction context") - } - - return rspCtx, err + return nil } diff --git a/glusterd2/transaction/rpc-service.go b/glusterd2/transaction/rpc-service.go index 2af8f596c..2edc2aa4e 100644 --- a/glusterd2/transaction/rpc-service.go +++ b/glusterd2/transaction/rpc-service.go @@ -45,14 +45,6 @@ func (p *txnSvc) RunStep(rpcCtx context.Context, req *TxnStepReq) (*TxnStepResp, if err != nil { logger.WithError(err).Debug("step function failed") resp.Error = err.Error() - } else { - b, err := json.Marshal(ctx) - if err != nil { - logger.WithError(err).Debug("failed to JSON marshal transcation context") - resp.Error = err.Error() - } else { - resp.Resp = b - } } return resp, nil diff --git a/glusterd2/transaction/step.go b/glusterd2/transaction/step.go index 012980106..677808463 100644 --- a/glusterd2/transaction/step.go +++ b/glusterd2/transaction/step.go @@ -76,7 +76,7 @@ func runStepFuncOnNode(name string, c TxnCtx, node uuid.UUID, done chan<- error) if uuid.Equal(node, gdctx.MyUUID) { done <- runStepFuncLocal(name, c) } else { - done <- runStepFuncRemote(name, c, node) + done <- runStepOn(name, node, c) } } @@ -90,10 +90,3 @@ func runStepFuncLocal(name string, c TxnCtx) error { return stepFunc(c) //TODO: Results need to be aggregated } - -func runStepFuncRemote(step string, c TxnCtx, node uuid.UUID) error { - rsp, err := runStepOn(step, node, c) - //TODO: Results need to be aggregated - _ = rsp - return err -} diff --git a/glusterd2/transaction/transaction-rpc.pb.go b/glusterd2/transaction/transaction-rpc.pb.go index 6a06fb30f..2ff286ac6 100644 --- a/glusterd2/transaction/transaction-rpc.pb.go +++ b/glusterd2/transaction/transaction-rpc.pb.go @@ -1,12 +1,11 @@ -// Code generated by protoc-gen-go. -// source: transaction/transaction-rpc.proto -// DO NOT EDIT! +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: transaction-rpc.proto /* Package transaction is a generated protocol buffer package. It is generated from these files: - transaction/transaction-rpc.proto + transaction-rpc.proto It has these top-level messages: TxnStepReq @@ -60,7 +59,6 @@ func (m *TxnStepReq) GetContext() []byte { type TxnStepResp struct { Error string `protobuf:"bytes,1,opt,name=Error" json:"Error,omitempty"` - Resp []byte `protobuf:"bytes,2,opt,name=Resp,proto3" json:"Resp,omitempty"` } func (m *TxnStepResp) Reset() { *m = TxnStepResp{} } @@ -75,13 +73,6 @@ func (m *TxnStepResp) GetError() string { return "" } -func (m *TxnStepResp) GetResp() []byte { - if m != nil { - return m.Resp - } - return nil -} - func init() { proto.RegisterType((*TxnStepReq)(nil), "transaction.TxnStepReq") proto.RegisterType((*TxnStepResp)(nil), "transaction.TxnStepResp") @@ -156,22 +147,22 @@ var _TxnSvc_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "transaction/transaction-rpc.proto", + Metadata: "transaction-rpc.proto", } -func init() { proto.RegisterFile("transaction/transaction-rpc.proto", fileDescriptor0) } +func init() { proto.RegisterFile("transaction-rpc.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x52, 0x2c, 0x29, 0x4a, 0xcc, - 0x2b, 0x4e, 0x4c, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x47, 0x62, 0xeb, 0x16, 0x15, 0x24, 0xeb, 0x15, - 0x14, 0xe5, 0x97, 0xe4, 0x0b, 0x71, 0x23, 0x09, 0x2b, 0x39, 0x71, 0x71, 0x85, 0x54, 0xe4, 0x05, - 0x97, 0xa4, 0x16, 0x04, 0xa5, 0x16, 0x0a, 0x49, 0x71, 0x71, 0x80, 0x98, 0x6e, 0xa5, 0x79, 0xc9, - 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x70, 0xbe, 0x90, 0x04, 0x17, 0xbb, 0x73, 0x7e, 0x5e, - 0x49, 0x6a, 0x45, 0x89, 0x04, 0x13, 0x50, 0x8a, 0x27, 0x08, 0xc6, 0x55, 0x32, 0xe7, 0xe2, 0x86, - 0x9b, 0x51, 0x5c, 0x20, 0x24, 0xc2, 0xc5, 0xea, 0x5a, 0x54, 0x94, 0x5f, 0x04, 0x35, 0x01, 0xc2, - 0x11, 0x12, 0xe2, 0x62, 0x01, 0xc9, 0x42, 0xf5, 0x82, 0xd9, 0x46, 0x1e, 0x5c, 0x6c, 0x20, 0x8d, - 0x65, 0xc9, 0x42, 0x76, 0x5c, 0xec, 0x41, 0xa5, 0x60, 0x23, 0x84, 0xc4, 0xf5, 0x90, 0xdc, 0xa7, - 0x87, 0x70, 0x9c, 0x94, 0x04, 0x76, 0x89, 0xe2, 0x02, 0x25, 0x86, 0x24, 0x36, 0xb0, 0xd7, 0x8c, - 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x0b, 0xfc, 0x47, 0xff, 0x00, 0x00, 0x00, + // 162 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2d, 0x29, 0x4a, 0xcc, + 0x2b, 0x4e, 0x4c, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x2d, 0x2a, 0x48, 0xd6, 0x2b, 0x28, 0xca, 0x2f, + 0xc9, 0x17, 0xe2, 0x46, 0x12, 0x56, 0x72, 0xe2, 0xe2, 0x0a, 0xa9, 0xc8, 0x0b, 0x2e, 0x49, 0x2d, + 0x08, 0x4a, 0x2d, 0x14, 0x92, 0xe2, 0xe2, 0x00, 0x31, 0xdd, 0x4a, 0xf3, 0x92, 0x25, 0x18, 0x15, + 0x18, 0x35, 0x38, 0x83, 0xe0, 0x7c, 0x21, 0x09, 0x2e, 0x76, 0xe7, 0xfc, 0xbc, 0x92, 0xd4, 0x8a, + 0x12, 0x09, 0x26, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x18, 0x57, 0x49, 0x99, 0x8b, 0x1b, 0x6e, 0x46, + 0x71, 0x81, 0x90, 0x08, 0x17, 0xab, 0x6b, 0x51, 0x51, 0x7e, 0x11, 0xd4, 0x04, 0x08, 0xc7, 0xc8, + 0x83, 0x8b, 0x0d, 0xa4, 0xa8, 0x2c, 0x59, 0xc8, 0x8e, 0x8b, 0x3d, 0xa8, 0x14, 0xac, 0x5c, 0x48, + 0x5c, 0x0f, 0xc9, 0x2d, 0x7a, 0x08, 0x87, 0x48, 0x49, 0x60, 0x97, 0x28, 0x2e, 0x50, 0x62, 0x48, + 0x62, 0x03, 0x7b, 0xc3, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x0e, 0x65, 0x9b, 0x63, 0xdf, 0x00, + 0x00, 0x00, } diff --git a/glusterd2/transaction/transaction-rpc.proto b/glusterd2/transaction/transaction-rpc.proto index d938b1d11..c84fafff2 100644 --- a/glusterd2/transaction/transaction-rpc.proto +++ b/glusterd2/transaction/transaction-rpc.proto @@ -9,7 +9,6 @@ message TxnStepReq { message TxnStepResp { string Error = 1; - bytes Resp = 2; // Resp is JSON encoded TxnCtx } service TxnSvc { From bbe034b9e42cd795a4a7e366e77307bad0a08e18 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Thu, 17 May 2018 15:54:57 +0530 Subject: [PATCH 30/54] txn: Aggregate results Signed-off-by: Prashanth Pai --- glusterd2/transaction/step.go | 75 ++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/glusterd2/transaction/step.go b/glusterd2/transaction/step.go index 677808463..0fe125d2d 100644 --- a/glusterd2/transaction/step.go +++ b/glusterd2/transaction/step.go @@ -2,10 +2,12 @@ package transaction import ( "errors" + "fmt" "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/pborman/uuid" + log "github.com/sirupsen/logrus" ) // StepFunc is the function that is supposed to be run during a transaction step @@ -41,16 +43,19 @@ func (s *Step) undo(c TxnCtx) error { return nil } -func runStepFuncOnNodes(name string, c TxnCtx, nodes []uuid.UUID) error { - var ( - i int - node uuid.UUID - ) - done := make(chan error) - defer close(done) +type stepResp struct { + node uuid.UUID + step string + err error +} + +func runStepFuncOnNodes(stepName string, c TxnCtx, nodes []uuid.UUID) error { + + respCh := make(chan stepResp, len(nodes)) + defer close(respCh) - for i, node = range nodes { - go runStepFuncOnNode(name, c, node, done) + for _, node := range nodes { + go runStepFuncOnNode(stepName, c, node, respCh) } // Ideally, we have to cancel the pending go-routines on first error @@ -58,35 +63,43 @@ func runStepFuncOnNodes(name string, c TxnCtx, nodes []uuid.UUID) error { // to do. Serializing sequentially is the easiest fix but we lose // concurrency. Instead, we let the do() function run on all nodes. - var err, lastErr error - for i >= 0 { - err = <-done - if err != nil { - // TODO: Need to properly aggregate results and - // check which node returned which error response. - lastErr = err + errCount := 0 + var resp stepResp + for range nodes { + resp = <-respCh + if resp.err != nil { + errCount++ + c.Logger().WithFields(log.Fields{ + "step": resp.step, "node": resp.node, + }).WithError(resp.err).Error("Step failed on node.") } - i-- } - return lastErr + if errCount != 0 { + return fmt.Errorf("Step %s failed on %d nodes", stepName, errCount) + } + + return nil } -func runStepFuncOnNode(name string, c TxnCtx, node uuid.UUID, done chan<- error) { +func runStepFuncOnNode(stepName string, c TxnCtx, node uuid.UUID, respCh chan<- stepResp) { + + c.Logger().WithFields(log.Fields{ + "step": stepName, "node": node, + }).Debug("Running step on node.") + + var err error if uuid.Equal(node, gdctx.MyUUID) { - done <- runStepFuncLocal(name, c) + // this (local) node + if stepFunc, ok := getStepFunc(stepName); ok { + err = stepFunc(c) + } else { + err = ErrStepFuncNotFound + } } else { - done <- runStepOn(name, node, c) + // remote node + err = runStepOn(stepName, node, c) } -} - -func runStepFuncLocal(name string, c TxnCtx) error { - c.Logger().WithField("stepfunc", name).Debug("running step function") - stepFunc, ok := getStepFunc(name) - if !ok { - return ErrStepFuncNotFound - } - return stepFunc(c) - //TODO: Results need to be aggregated + respCh <- stepResp{node, stepName, err} } From 574e69f13259bb4415e3fcb9e28f8a9d38375e2d Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Thu, 24 May 2018 12:31:43 +0530 Subject: [PATCH 31/54] Add reserved keyword for removed field in TxnStepResp Reference: https://developers.google.com/protocol-buffers/docs/proto3#reserved Signed-off-by: Prashanth Pai --- glusterd2/transaction/transaction-rpc.pb.go | 14 +++++++------- glusterd2/transaction/transaction-rpc.proto | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/glusterd2/transaction/transaction-rpc.pb.go b/glusterd2/transaction/transaction-rpc.pb.go index 2ff286ac6..cc581e4e6 100644 --- a/glusterd2/transaction/transaction-rpc.pb.go +++ b/glusterd2/transaction/transaction-rpc.pb.go @@ -153,16 +153,16 @@ var _TxnSvc_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("transaction-rpc.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 162 bytes of a gzipped FileDescriptorProto + // 172 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2d, 0x29, 0x4a, 0xcc, 0x2b, 0x4e, 0x4c, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x2d, 0x2a, 0x48, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x46, 0x12, 0x56, 0x72, 0xe2, 0xe2, 0x0a, 0xa9, 0xc8, 0x0b, 0x2e, 0x49, 0x2d, 0x08, 0x4a, 0x2d, 0x14, 0x92, 0xe2, 0xe2, 0x00, 0x31, 0xdd, 0x4a, 0xf3, 0x92, 0x25, 0x18, 0x15, 0x18, 0x35, 0x38, 0x83, 0xe0, 0x7c, 0x21, 0x09, 0x2e, 0x76, 0xe7, 0xfc, 0xbc, 0x92, 0xd4, 0x8a, - 0x12, 0x09, 0x26, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x18, 0x57, 0x49, 0x99, 0x8b, 0x1b, 0x6e, 0x46, - 0x71, 0x81, 0x90, 0x08, 0x17, 0xab, 0x6b, 0x51, 0x51, 0x7e, 0x11, 0xd4, 0x04, 0x08, 0xc7, 0xc8, - 0x83, 0x8b, 0x0d, 0xa4, 0xa8, 0x2c, 0x59, 0xc8, 0x8e, 0x8b, 0x3d, 0xa8, 0x14, 0xac, 0x5c, 0x48, - 0x5c, 0x0f, 0xc9, 0x2d, 0x7a, 0x08, 0x87, 0x48, 0x49, 0x60, 0x97, 0x28, 0x2e, 0x50, 0x62, 0x48, - 0x62, 0x03, 0x7b, 0xc3, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x0e, 0x65, 0x9b, 0x63, 0xdf, 0x00, - 0x00, 0x00, + 0x12, 0x09, 0x26, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x18, 0x57, 0x49, 0x9f, 0x8b, 0x1b, 0x6e, 0x46, + 0x71, 0x81, 0x90, 0x08, 0x17, 0xab, 0x6b, 0x51, 0x51, 0x7e, 0x11, 0xd4, 0x04, 0x08, 0xc7, 0x8b, + 0x85, 0x83, 0x49, 0x80, 0x39, 0x88, 0x05, 0xa4, 0xc2, 0xc8, 0x83, 0x8b, 0x0d, 0xa4, 0xa1, 0x2c, + 0x59, 0xc8, 0x8e, 0x8b, 0x3d, 0xa8, 0x14, 0xac, 0x55, 0x48, 0x5c, 0x0f, 0xc9, 0x5d, 0x7a, 0x08, + 0x47, 0x49, 0x49, 0x60, 0x97, 0x28, 0x2e, 0x50, 0x62, 0x48, 0x62, 0x03, 0x7b, 0xc9, 0x18, 0x10, + 0x00, 0x00, 0xff, 0xff, 0x70, 0x0d, 0x1c, 0x06, 0xeb, 0x00, 0x00, 0x00, } diff --git a/glusterd2/transaction/transaction-rpc.proto b/glusterd2/transaction/transaction-rpc.proto index c84fafff2..164fc57fe 100644 --- a/glusterd2/transaction/transaction-rpc.proto +++ b/glusterd2/transaction/transaction-rpc.proto @@ -8,6 +8,8 @@ message TxnStepReq { } message TxnStepResp { + reserved 2; + reserved "Resp"; string Error = 1; } From 049ba9f8ed9894de9905ba3acc47532195709186 Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Thu, 24 May 2018 14:49:36 +0530 Subject: [PATCH 32/54] config: removed default configuration file To view the default values run `glusterd2 --help` Signed-off-by: Aravinda VK --- .gitignore | 2 - Makefile | 27 +++++------- extras/make/paths.mk | 1 + glusterd2/config.go | 80 ++++++++++++---------------------- scripts/build.sh | 4 ++ scripts/gen-gd2conf.sh | 16 ------- scripts/prepare_path_config.sh | 13 ------ 7 files changed, 44 insertions(+), 99 deletions(-) delete mode 100755 scripts/prepare_path_config.sh diff --git a/.gitignore b/.gitignore index 911d8b86f..c5e6301d9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,3 @@ endpoints.json generate-doc # Ignore vi backup files *~ -# Generated by Make -glusterd2/paths_config.go diff --git a/Makefile b/Makefile index ef7612e72..ef8b9d540 100644 --- a/Makefile +++ b/Makefile @@ -24,13 +24,14 @@ GD2CONF_INSTALL = $(DESTDIR)$(SYSCONFDIR)/$(GD2)/$(GD2_CONF) GD2STATEDIR = $(LOCALSTATEDIR)/$(GD2) GD2LOGDIR = $(LOGDIR)/$(GD2) +GD2RUNDIR = $(RUNDIR)/$(GD2) DEPENV ?= PLUGINS ?= yes FASTBUILD ?= yes -.PHONY: all build check check-go check-reqs install vendor-update vendor-install verify release check-protoc $(GD2_BIN) $(GD2_BUILD) $(CLI_BIN) $(CLI_BUILD) cli $(GD2_CONF) gd2conf test dist dist-vendor functest generated +.PHONY: all build check check-go check-reqs install vendor-update vendor-install verify release check-protoc $(GD2_BIN) $(GD2_BUILD) $(CLI_BIN) $(CLI_BUILD) cli $(GD2_CONF) gd2conf test dist dist-vendor functest all: build @@ -49,17 +50,9 @@ check-reqs: @./scripts/check-reqs.sh @echo -generated: - @echo "Generating sources..." - @PREFIX=$(PREFIX) BASE_PREFIX=$(BASE_PREFIX) EXEC_PREFIX=$(EXEC_PREFIX) \ - BINDIR=$(BINDIR) SBINDIR=$(SBINDIR) DATADIR=$(DATADIR) \ - LOCALSTATEDIR=$(LOCALSTATEDIR) LOGDIR=$(LOGDIR) \ - SYSCONFDIR=$(SYSCONFDIR) ./scripts/prepare_path_config.sh glusterd2 - - -$(GD2_BIN): $(GD2_BUILD) gd2conf generated -$(GD2_BUILD): generated - @PLUGINS=$(PLUGINS) FASTBUILD=$(FASTBUILD) ./scripts/build.sh glusterd2 +$(GD2_BIN): $(GD2_BUILD) gd2conf +$(GD2_BUILD): + @PLUGINS=$(PLUGINS) FASTBUILD=$(FASTBUILD) BASE_PREFIX=$(BASE_PREFIX) ./scripts/build.sh glusterd2 @echo $(CLI_BIN) cli: $(CLI_BUILD) @@ -69,9 +62,9 @@ $(CLI_BUILD): @./$(CLI_BASH_COMPLETION_GEN_BIN) $(CLI_BASH_COMPLETION_BUILD) @echo -$(GD2_CONF) gd2conf: $(GD2CONF_BUILD) -$(GD2CONF_BUILD): - @GD2STATEDIR=$(GD2STATEDIR) GD2LOGDIR=$(GD2LOGDIR) $(GD2CONF_BUILDSCRIPT) +$(GD2_CONF) gd2conf: + @GD2=$(GD2) GD2STATEDIR=$(GD2STATEDIR) GD2LOGDIR=$(GD2LOGDIR) \ + GD2RUNDIR=$(GD2RUNDIR) $(GD2CONF_BUILDSCRIPT) install: install -D $(GD2_BUILD) $(GD2_INSTALL) @@ -90,10 +83,10 @@ vendor-install: @$(DEPENV) dep ensure @echo -test: check-reqs generated +test: check-reqs @./test.sh $(TESTOPTIONS) -functest: check-reqs generated +functest: check-reqs @go test ./e2e -v -functest release: build diff --git a/extras/make/paths.mk b/extras/make/paths.mk index 07db1dc8e..b8fa060ca 100644 --- a/extras/make/paths.mk +++ b/extras/make/paths.mk @@ -21,3 +21,4 @@ LOCALSTATEDIR ?= $(BASE_PREFIX)/var/lib LOGDIR ?= $(BASE_PREFIX)/var/log SYSCONFDIR ?= $(BASE_PREFIX)/etc +RUNDIR ?= $(BASE_PREFIX)/var/run diff --git a/glusterd2/config.go b/glusterd2/config.go index 95d3d7153..fa4402f9c 100644 --- a/glusterd2/config.go +++ b/glusterd2/config.go @@ -7,7 +7,6 @@ import ( "net" "os" "path" - "path/filepath" "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/store" @@ -18,39 +17,49 @@ import ( config "github.com/spf13/viper" ) +const ( + defaultlogfile = "STDOUT" + defaultpeerport = "24008" + defaultpeeraddress = ":24008" + defaultclientaddress = ":24007" + defaultloglevel = "debug" +) + var ( // metrics expConfig = expvar.NewMap("config") -) -const ( - defaultLogLevel = "debug" - defaultConfName = "glusterd2" + // defaultPathPrefix is set by LDFLAGS + defaultPathPrefix = "" + + defaultlocalstatedir = defaultPathPrefix + "/var/lib/glusterd2" + defaultlogdir = defaultPathPrefix + "/var/log/glusterd2" + defaultrundir = defaultPathPrefix + "/var/run/glusterd2" ) // parseFlags sets up the flags and parses them, this needs to be called before any other operation func parseFlags() { flag.String("workdir", "", "Working directory for GlusterD. (default: current directory)") - flag.String("localstatedir", "", "Directory to store local state information. (default: workdir)") - flag.String("rundir", "", "Directory to store runtime data.") - flag.String("config", "", "Configuration file for GlusterD. By default looks for glusterd2.toml in [/usr/local]/etc/glusterd2 and current working directory.") + flag.String("localstatedir", defaultlocalstatedir, "Directory to store local state information.") + flag.String("rundir", defaultrundir, "Directory to store runtime data.") + flag.String("config", "", "Configuration file for GlusterD.") - flag.String(logging.DirFlag, "", logging.DirHelp+" (default: workdir/log)") - flag.String(logging.FileFlag, "STDOUT", logging.FileHelp) - flag.String(logging.LevelFlag, defaultLogLevel, logging.LevelHelp) + flag.String(logging.DirFlag, defaultlogdir, logging.DirHelp) + flag.String(logging.FileFlag, defaultlogfile, logging.FileHelp) + flag.String(logging.LevelFlag, defaultloglevel, logging.LevelHelp) // TODO: Change default to false (disabled) in future. flag.Bool("statedump", true, "Enable /statedump endpoint for metrics.") - flag.String("clientaddress", "", "Address to bind the REST service.") - flag.String("peeraddress", "", "Address to bind the inter glusterd2 RPC service.") + flag.String("clientaddress", defaultclientaddress, "Address to bind the REST service.") + flag.String("peeraddress", defaultpeeraddress, "Address to bind the inter glusterd2 RPC service.") // TODO: SSL/TLS is currently only implemented for REST interface flag.String("cert-file", "", "Certificate used for SSL/TLS connections from clients to glusterd2.") flag.String("key-file", "", "Private key for the SSL/TLS certificate.") // PID file - flag.String("pidfile", "", "PID file path(default: rundir/gluster/glusterd2.pid)") + flag.String("pidfile", "", "PID file path. (default \"rundir/glusterd2.pid)\"") store.InitFlags() @@ -65,22 +74,12 @@ func setDefaults() error { return err } - wd := config.GetString("workdir") - if wd == "" { + if config.GetString("workdir") == "" { config.SetDefault("workdir", cwd) - wd = cwd - } - - if config.GetString("localstatedir") == "" { - config.SetDefault("localstatedir", wd) } config.SetDefault("hooksdir", config.GetString("localstatedir")+"/hooks") - if config.GetString(logging.DirFlag) == "" { - config.SetDefault(logging.DirFlag, path.Join(config.GetString("localstatedir"), "log")) - } - if config.GetString("pidfile") == "" { config.SetDefault("pidfile", path.Join(config.GetString("rundir"), "glusterd2.pid")) } @@ -94,7 +93,7 @@ func setDefaults() error { host = gdctx.HostIP } if port == "" { - port = config.GetString("defaultpeerport") + port = defaultpeerport } config.Set("peeraddress", host+":"+port) @@ -111,7 +110,10 @@ func (v valueType) String() string { } func dumpConfigToLog() { - log.WithField("file", config.ConfigFileUsed()).Info("loaded configuration from file") + if config.ConfigFileUsed() != "" { + log.WithField("file", config.ConfigFileUsed()).Info("loaded configuration from file") + } + l := log.NewEntry(log.StandardLogger()) for k, v := range config.AllSettings() { @@ -129,30 +131,6 @@ func initConfig(confFile string) error { // Limit config to toml only to avoid confusion with multiple config types config.SetConfigType("toml") - config.SetConfigName(defaultConfName) - - // Set default config dir and path if default config file exists - // Chances of not having default config file is only during development - // Add current directory to this path if default conf file exists - confdir := defaultConfDir - conffile := defaultConfDir + "/" + defaultConfName + ".toml" - if _, err := os.Stat(defaultConfDir + "/" + defaultConfName + ".toml"); os.IsNotExist(err) { - cdir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return err - } - confdir = cdir - conffile = cdir + "/" + defaultConfName + ".toml" - log.Info("default config file not found, loading config file from current directory") - } - - config.AddConfigPath(confdir) - if err := config.MergeInConfig(); err != nil { - log.WithError(err). - WithField("file", conffile). - Error("failed to read default config file") - return err - } // If custom configuration is passed if confFile != "" { diff --git a/scripts/build.sh b/scripts/build.sh index 204b3e615..7895e4178 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -34,6 +34,10 @@ GIT_SHA_FULL=${GIT_SHA_FULL:-$(git rev-parse HEAD || echo "undefined")} LDFLAGS="-X ${REPO_PATH}/version.GlusterdVersion=${VERSION} -X ${REPO_PATH}/version.GitSHA=${GIT_SHA}" LDFLAGS+=" -B 0x${GIT_SHA_FULL}" +if [ "$BIN" == "glusterd2" ]; then + LDFLAGS+=" -X main.defaultPathPrefix=${BASE_PREFIX}" +fi + GOBUILD_TAGS="" if [ "$PLUGINS" == "yes" ]; then GOBUILD_TAGS+="plugins " diff --git a/scripts/gen-gd2conf.sh b/scripts/gen-gd2conf.sh index 96f9c25db..7092d884d 100755 --- a/scripts/gen-gd2conf.sh +++ b/scripts/gen-gd2conf.sh @@ -1,21 +1,5 @@ #!/bin/bash -PREFIX=${PREFIX:-/usr/local} -BASE_PREFIX=$PREFIX -if [ "$PREFIX" = "/usr" ]; then - BASE_PREFIX="" -fi - -DATADIR=${DATADIR:-$PREFIX/share} -LOCALSTATEDIR=${LOCALSTATEDIR:-$PREFIX/var/lib} -LOGDIR=${LOGDIR:-$BASE_PREFIX/var/log} -RUNDIR=${RUNDIR:-$BASE_PREFIX/var/run} - -GD2="glusterd2" -GD2STATEDIR=${GD2STATEDIR:-$LOCALSTATEDIR/$GD2} -GD2LOGDIR=${GD2LOGDIR:-$LOGDIR/$GD2} -GD2RUNDIR=${GD2RUNDIR:-$RUNDIR/$GD2} - OUTDIR=${1:-build} mkdir -p "$OUTDIR" diff --git a/scripts/prepare_path_config.sh b/scripts/prepare_path_config.sh deleted file mode 100755 index d3a9c133e..000000000 --- a/scripts/prepare_path_config.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Create glusterd2/paths_config.go -glusterd2dir=$1 - -OUTPUT=${glusterd2dir}/paths_config.go - - -cat >"$OUTPUT" < Date: Thu, 24 May 2018 10:28:25 -0400 Subject: [PATCH 33/54] Changing api path --- plugins/device/api/resp.go | 7 ------- plugins/device/init.go | 7 ------- plugins/device/rest.go | 2 -- 3 files changed, 16 deletions(-) diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index 4530eb4b9..ce085e249 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -15,10 +15,3 @@ type AddDeviceResp api.Peer // ListDeviceResp is the success response sent to a ListDevice request type ListDeviceResp []Info - -// ListAllDeviceResp is the success response sent to a ListAllDevice request -type ListAllDeviceResp struct { - Name string `json:"name"` - State string `json:"state"` - Peerid string `json:"peerid,omitempty"` -} diff --git a/plugins/device/init.go b/plugins/device/init.go index d58eb0692..029b7f04c 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -34,13 +34,6 @@ func (p *Plugin) RestRoutes() route.Routes { Version: 1, ResponseType: utils.GetTypeString((*deviceapi.ListDeviceResp)(nil)), HandlerFunc: deviceListHandler}, - route.Route{ - Name: "ListAllDevices", - Method: "GET", - Pattern: "/devices", - Version: 1, - ResponseType: utils.GetTypeString((*[]deviceapi.ListDeviceResp)(nil)), - HandlerFunc: listAllDevicesHandler}, } } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 02c92e1ef..f35f7af4d 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -141,8 +141,6 @@ func listAllHandler(w http.ResponseWriter, r *http.Request) { logger.WithError(err).Error("Failed to get peers from store") } - for key := range peers { - devices, err := GetDevices(peerID) if err != nil { logger.WithError(err).WithField("peerid", peerID).Error("Failed to get devices for peer") From e613a8730ce5179ec96ffa19a54e57d207f85ffe Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Tue, 17 Apr 2018 12:51:51 +0530 Subject: [PATCH 34/54] Refactoring for Intelligent Volume Provisioning Added more device utils and `fstab` handling library Updates: #466 Signed-off-by: Aravinda VK --- glusterd2/commands/volumes/events.go | 26 --- .../volumes/volume-create-disperse.go | 16 +- glusterd2/commands/volumes/volume-create.go | 4 +- glusterd2/commands/volumes/volume-delete.go | 2 +- glusterd2/commands/volumes/volume-expand.go | 4 +- glusterd2/commands/volumes/volume-start.go | 2 +- glusterd2/commands/volumes/volume-stop.go | 2 +- glusterd2/volume/events.go | 32 ++++ glusterd2/volume/volume-utils.go | 14 ++ .../volumes => pkg/api}/brickutils.go | 10 +- plugins/device/api/resp.go | 8 +- plugins/device/deviceutils/fstab.go | 168 ++++++++++++++++++ .../device/{ => deviceutils}/store-utils.go | 45 ++++- plugins/device/deviceutils/utils.go | 109 ++++++++++++ plugins/device/rest.go | 3 +- plugins/device/transaction.go | 12 +- 16 files changed, 395 insertions(+), 62 deletions(-) delete mode 100644 glusterd2/commands/volumes/events.go create mode 100644 glusterd2/volume/events.go rename {glusterd2/commands/volumes => pkg/api}/brickutils.go (76%) create mode 100644 plugins/device/deviceutils/fstab.go rename plugins/device/{ => deviceutils}/store-utils.go (54%) diff --git a/glusterd2/commands/volumes/events.go b/glusterd2/commands/volumes/events.go deleted file mode 100644 index fdc5668d8..000000000 --- a/glusterd2/commands/volumes/events.go +++ /dev/null @@ -1,26 +0,0 @@ -package volumecommands - -import ( - "github.com/gluster/glusterd2/glusterd2/events" - "github.com/gluster/glusterd2/glusterd2/volume" - "github.com/gluster/glusterd2/pkg/api" -) - -type volumeEvent string - -const ( - eventVolumeCreated volumeEvent = "volume.created" - eventVolumeExpanded = "volume.expanded" - eventVolumeStarted = "volume.started" - eventVolumeStopped = "volume.stopped" - eventVolumeDeleted = "volume.deleted" -) - -func newVolumeEvent(e volumeEvent, v *volume.Volinfo) *api.Event { - data := map[string]string{ - "volume.name": v.Name, - "volume.id": v.ID.String(), - } - - return events.New(string(e), data, true) -} diff --git a/glusterd2/commands/volumes/volume-create-disperse.go b/glusterd2/commands/volumes/volume-create-disperse.go index a7a35ee7d..e03062164 100644 --- a/glusterd2/commands/volumes/volume-create-disperse.go +++ b/glusterd2/commands/volumes/volume-create-disperse.go @@ -7,20 +7,6 @@ import ( "github.com/gluster/glusterd2/pkg/api" ) -func getRedundancy(disperse uint) uint { - var temp, l, mask uint - temp = disperse - l = 0 - for temp = temp >> 1; temp != 0; temp = temp >> 1 { - l = l + 1 - } - mask = ^(1 << l) - if red := disperse & mask; red != 0 { - return red - } - return 1 -} - func checkDisperseParams(req *api.SubvolReq, s *volume.Subvol) error { count := len(req.Bricks) @@ -50,7 +36,7 @@ func checkDisperseParams(req *api.SubvolReq, s *volume.Subvol) error { } if req.DisperseRedundancy <= 0 { - req.DisperseRedundancy = int(getRedundancy(uint(req.DisperseCount))) + req.DisperseRedundancy = volume.GetRedundancy(uint(req.DisperseCount)) } if req.DisperseCount != count { diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index 5881b328a..d9955737d 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -110,7 +110,7 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { return } - nodes, err := nodesFromVolumeCreateReq(&req) + nodes, err := req.Nodes() if err != nil { restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) return @@ -172,7 +172,7 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { } logger.WithField("volume-name", volinfo.Name).Info("new volume created") - events.Broadcast(newVolumeEvent(eventVolumeCreated, volinfo)) + events.Broadcast(volume.NewEvent(volume.EventVolumeCreated, volinfo)) resp := createVolumeCreateResp(volinfo) restutils.SendHTTPResponse(ctx, w, http.StatusCreated, resp) diff --git a/glusterd2/commands/volumes/volume-delete.go b/glusterd2/commands/volumes/volume-delete.go index 06295acf5..06b35f38e 100644 --- a/glusterd2/commands/volumes/volume-delete.go +++ b/glusterd2/commands/volumes/volume-delete.go @@ -122,7 +122,7 @@ func volumeDeleteHandler(w http.ResponseWriter, r *http.Request) { } logger.WithField("volume-name", volinfo.Name).Info("volume deleted") - events.Broadcast(newVolumeEvent(eventVolumeDeleted, volinfo)) + events.Broadcast(volume.NewEvent(volume.EventVolumeDeleted, volinfo)) restutils.SendHTTPResponse(ctx, w, http.StatusOK, nil) } diff --git a/glusterd2/commands/volumes/volume-expand.go b/glusterd2/commands/volumes/volume-expand.go index a7a4ebbd6..e8a190253 100644 --- a/glusterd2/commands/volumes/volume-expand.go +++ b/glusterd2/commands/volumes/volume-expand.go @@ -95,7 +95,7 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { txn := transaction.NewTxn(ctx) defer txn.Cleanup() - nodes, err := nodesFromVolumeExpandReq(&req) + nodes, err := req.Nodes() if err != nil { logger.WithError(err).Error("could not prepare node list") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) @@ -170,7 +170,7 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { } logger.WithField("volume-name", volinfo.Name).Info("volume expanded") - events.Broadcast(newVolumeEvent(eventVolumeExpanded, volinfo)) + events.Broadcast(volume.NewEvent(volume.EventVolumeExpanded, volinfo)) resp := createVolumeExpandResp(volinfo) restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) diff --git a/glusterd2/commands/volumes/volume-start.go b/glusterd2/commands/volumes/volume-start.go index 34658fe50..c13aca2eb 100644 --- a/glusterd2/commands/volumes/volume-start.go +++ b/glusterd2/commands/volumes/volume-start.go @@ -136,7 +136,7 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { } logger.WithField("volume-name", volinfo.Name).Info("volume started") - events.Broadcast(newVolumeEvent(eventVolumeStarted, volinfo)) + events.Broadcast(volume.NewEvent(volume.EventVolumeStarted, volinfo)) resp := createVolumeStartResp(volinfo) restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) diff --git a/glusterd2/commands/volumes/volume-stop.go b/glusterd2/commands/volumes/volume-stop.go index 012e032d8..4809209c7 100644 --- a/glusterd2/commands/volumes/volume-stop.go +++ b/glusterd2/commands/volumes/volume-stop.go @@ -133,7 +133,7 @@ func volumeStopHandler(w http.ResponseWriter, r *http.Request) { } logger.WithField("volume-name", volinfo.Name).Info("volume stopped") - events.Broadcast(newVolumeEvent(eventVolumeStopped, volinfo)) + events.Broadcast(volume.NewEvent(volume.EventVolumeStopped, volinfo)) resp := createVolumeStopResp(volinfo) restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) diff --git a/glusterd2/volume/events.go b/glusterd2/volume/events.go new file mode 100644 index 000000000..b43ee2f6d --- /dev/null +++ b/glusterd2/volume/events.go @@ -0,0 +1,32 @@ +package volume + +import ( + "github.com/gluster/glusterd2/glusterd2/events" + "github.com/gluster/glusterd2/pkg/api" +) + +// Event represents Volume life cycle events +type Event string + +const ( + // EventVolumeCreated represents Volume Create event + EventVolumeCreated Event = "volume.created" + // EventVolumeExpanded represents Volume Expand event + EventVolumeExpanded = "volume.expanded" + // EventVolumeStarted represents Volume Start event + EventVolumeStarted = "volume.started" + // EventVolumeStopped represents Volume Stop event + EventVolumeStopped = "volume.stopped" + // EventVolumeDeleted represents Volume Delete event + EventVolumeDeleted = "volume.deleted" +) + +// NewEvent adds required details to event based on Volume info +func NewEvent(e Event, v *Volinfo) *api.Event { + data := map[string]string{ + "volume.name": v.Name, + "volume.id": v.ID.String(), + } + + return events.New(string(e), data, true) +} diff --git a/glusterd2/volume/volume-utils.go b/glusterd2/volume/volume-utils.go index 51d695d36..0cef00e62 100644 --- a/glusterd2/volume/volume-utils.go +++ b/glusterd2/volume/volume-utils.go @@ -17,6 +17,20 @@ import ( log "github.com/sirupsen/logrus" ) +// GetRedundancy calculates redundancy count based on disperse count +func GetRedundancy(disperse uint) int { + var temp, l, mask uint + temp = disperse + for temp = temp >> 1; temp != 0; temp = temp >> 1 { + l = l + 1 + } + mask = ^(1 << l) + if red := disperse & mask; red != 0 { + return int(red) + } + return 1 +} + // isBrickPathAvailable validates whether the brick is consumed by other // volume func isBrickPathAvailable(peerID uuid.UUID, brickPath string) error { diff --git a/glusterd2/commands/volumes/brickutils.go b/pkg/api/brickutils.go similarity index 76% rename from glusterd2/commands/volumes/brickutils.go rename to pkg/api/brickutils.go index 52927037f..28da271f4 100644 --- a/glusterd2/commands/volumes/brickutils.go +++ b/pkg/api/brickutils.go @@ -1,14 +1,13 @@ -package volumecommands +package api import ( "fmt" - "github.com/gluster/glusterd2/pkg/api" - "github.com/pborman/uuid" ) -func nodesFromVolumeCreateReq(req *api.VolCreateReq) ([]uuid.UUID, error) { +// Nodes extracts the list of nodes from Volume Create request +func (req *VolCreateReq) Nodes() ([]uuid.UUID, error) { var nodesMap = make(map[string]bool) var nodes []uuid.UUID for _, subvol := range req.Subvols { @@ -26,7 +25,8 @@ func nodesFromVolumeCreateReq(req *api.VolCreateReq) ([]uuid.UUID, error) { return nodes, nil } -func nodesFromVolumeExpandReq(req *api.VolExpandReq) ([]uuid.UUID, error) { +// Nodes extracts list of Peer IDs from Volume Expand request +func (req *VolExpandReq) Nodes() ([]uuid.UUID, error) { var nodesMap = make(map[string]bool) var nodes []uuid.UUID for _, brick := range req.Bricks { diff --git a/plugins/device/api/resp.go b/plugins/device/api/resp.go index fccda3874..ffb30166d 100644 --- a/plugins/device/api/resp.go +++ b/plugins/device/api/resp.go @@ -6,8 +6,12 @@ import ( // Info represents structure in which devices are to be store in Peer Metadata type Info struct { - Name string `json:"name"` - State string `json:"state"` + Name string `json:"name"` + State string `json:"state"` + VgName string `json:"vg-name"` + AvailableSize uint64 `json:"available-size"` + ExtentSize uint64 `json:"extent-size"` + Used bool `json:"used"` } // AddDeviceResp is the success response sent to a AddDeviceReq request diff --git a/plugins/device/deviceutils/fstab.go b/plugins/device/deviceutils/fstab.go new file mode 100644 index 000000000..ecbd49d5c --- /dev/null +++ b/plugins/device/deviceutils/fstab.go @@ -0,0 +1,168 @@ +package deviceutils + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + "syscall" +) + +// FstabMount represents entry in Fstab file +type FstabMount struct { + Device string + MountPoint string + FilesystemFormat string + MountOptions string + DumpValue string + FsckOption string +} + +// Fstab represents Fstab entries +type Fstab struct { + Filename string + Mounts []FstabMount +} + +func lock(filename string) (*os.File, error) { + lockfile, err := os.Create(filename + ".lock") + if err != nil { + return nil, err + } + + err = syscall.Flock(int(lockfile.Fd()), syscall.LOCK_EX) + if err != nil { + return nil, err + } + return lockfile, nil +} + +func (f *Fstab) save() error { + data := "\n" + for _, mnt := range f.Mounts { + data += fmt.Sprintf("%s %s %s %s %s %s\n", + mnt.Device, + mnt.MountPoint, + mnt.FilesystemFormat, + mnt.MountOptions, + mnt.DumpValue, + mnt.FsckOption, + ) + } + err := ioutil.WriteFile(f.Filename+".tmp", []byte(data), 0600) + if err != nil { + return err + } + return os.Rename(f.Filename+".tmp", f.Filename) +} + +func (f *Fstab) load() error { + out, err := ioutil.ReadFile(f.Filename) + if os.IsNotExist(err) { + return nil + } + + if err != nil { + return err + } + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + var parts []string + for _, p := range strings.Split(line, " ") { + if p != "" { + parts = append(parts, p) + } + } + + if len(parts) != 6 { + return errors.New("Invalid mount entry: " + line) + } + f.Mounts = append(f.Mounts, FstabMount{ + Device: parts[0], + MountPoint: parts[1], + FilesystemFormat: parts[2], + MountOptions: parts[3], + DumpValue: parts[4], + FsckOption: parts[5], + }) + } + return nil +} + +func (f *Fstab) mountExists(mountpoint string) bool { + for _, mnt := range f.Mounts { + if mnt.MountPoint == mountpoint { + return true + } + } + return false +} + +// FstabAddMount adds mount point entry to fstab +func FstabAddMount(filename string, mnt FstabMount) error { + fstab := Fstab{Filename: filename} + + file, err := lock(fstab.Filename) + if err != nil { + return err + } + + defer file.Close() + + err = fstab.load() + if err != nil { + return err + } + + if mnt.FilesystemFormat == "" { + mnt.FilesystemFormat = "xfs" + } + + if mnt.DumpValue == "" { + mnt.DumpValue = "0" + } + + if mnt.FsckOption == "" { + mnt.FsckOption = "0" + } + + if mnt.MountOptions == "" { + mnt.MountOptions = "defaults" + } + + if !fstab.mountExists(mnt.MountPoint) { + fstab.Mounts = append(fstab.Mounts, mnt) + } + return fstab.save() +} + +// FstabRemoveMount removes mountpoint from fstab +func FstabRemoveMount(filename, mountpoint string) error { + fstab := Fstab{Filename: filename} + file, err := lock(fstab.Filename) + if err != nil { + return err + } + + defer file.Close() + + err = fstab.load() + if err != nil { + return err + } + + var mounts []FstabMount + for _, mnt := range fstab.Mounts { + if mnt.MountPoint != mountpoint { + mounts = append(mounts, mnt) + } + } + fstab.Mounts = mounts + + return fstab.save() +} diff --git a/plugins/device/store-utils.go b/plugins/device/deviceutils/store-utils.go similarity index 54% rename from plugins/device/store-utils.go rename to plugins/device/deviceutils/store-utils.go index f593f8e99..2e7eadd31 100644 --- a/plugins/device/store-utils.go +++ b/plugins/device/deviceutils/store-utils.go @@ -1,4 +1,4 @@ -package device +package deviceutils import ( "encoding/json" @@ -23,7 +23,8 @@ func GetDevices(peerID string) ([]deviceapi.Info, error) { return nil, nil } -func checkIfDeviceExist(reqDevice string, devices []deviceapi.Info) bool { +// DeviceExist checks the given device existence +func DeviceExist(reqDevice string, devices []deviceapi.Info) bool { for _, key := range devices { if reqDevice == key.Name { return true @@ -32,7 +33,8 @@ func checkIfDeviceExist(reqDevice string, devices []deviceapi.Info) bool { return false } -func addDevice(device deviceapi.Info, peerID string) error { +// AddDevice adds device to peerinfo +func AddDevice(device deviceapi.Info, peerID string) error { deviceDetails, err := GetDevices(peerID) if err != nil { return err @@ -60,3 +62,40 @@ func addDevice(device deviceapi.Info, peerID string) error { return nil } + +// UpdateDeviceFreeSize updates the actual available size of VG +func UpdateDeviceFreeSize(peerid, vgname string) error { + deviceDetails, err := GetDevices(peerid) + if err != nil { + return err + } + + peerInfo, err := peer.GetPeer(peerid) + if err != nil { + return err + } + + for idx, dev := range deviceDetails { + if dev.VgName == vgname { + availableSize, extentSize, err := GetVgAvailableSize(vgname) + if err != nil { + return err + } + deviceDetails[idx].AvailableSize = availableSize + deviceDetails[idx].ExtentSize = extentSize + } + } + + deviceJSON, err := json.Marshal(deviceDetails) + if err != nil { + return err + } + + peerInfo.Metadata["_devices"] = string(deviceJSON) + err = peer.AddOrUpdatePeer(peerInfo) + if err != nil { + return err + } + + return nil +} diff --git a/plugins/device/deviceutils/utils.go b/plugins/device/deviceutils/utils.go index 7fad76717..449fbe208 100644 --- a/plugins/device/deviceutils/utils.go +++ b/plugins/device/deviceutils/utils.go @@ -1,7 +1,16 @@ package deviceutils import ( + "errors" + "fmt" "os/exec" + "strconv" + "strings" +) + +const ( + maxMetadataSizeGb = 16 + chunkSize = "1280k" ) //CreatePV is used to create physical volume. @@ -27,3 +36,103 @@ func RemovePV(device string) error { pvremoveCmd := exec.Command("pvremove", device) return pvremoveCmd.Run() } + +// MbToKb converts Value from Mb to Kb +func MbToKb(value uint64) uint64 { + return value * 1024 +} + +// GbToKb converts Value from Gb to Kb +func GbToKb(value uint64) uint64 { + return value * 1024 * 1024 +} + +// GetVgAvailableSize gets available size of given Vg +func GetVgAvailableSize(vgname string) (uint64, uint64, error) { + out, err := exec.Command("vgdisplay", "-c", vgname).Output() + if err != nil { + return 0, 0, err + } + vgdata := strings.Split(strings.TrimRight(string(out), "\n"), ":") + + if len(vgdata) != 17 { + return 0, 0, errors.New("Failed to get free size of VG: " + vgname) + } + + // Physical extent size index is 12 + extentSize, err := strconv.ParseUint(vgdata[12], 10, 64) + if err != nil { + return 0, 0, err + } + + // Free Extents index is 15 + freeExtents, err := strconv.ParseUint(vgdata[15], 10, 64) + if err != nil { + return 0, 0, err + } + + return extentSize * freeExtents, extentSize, nil +} + +// GetPoolMetadataSize calculates the thin pool metadata size based on the given thin pool size +func GetPoolMetadataSize(poolsize uint64) uint64 { + // https://access.redhat.com/documentation/en-us/red_hat_gluster_storage/3.3/html-single/administration_guide/#Brick_Configuration + // Minimum metadata size required is 0.5% and Max upto 16GB + + metadataSize := uint64(float64(poolsize) * 0.005) + if metadataSize > GbToKb(maxMetadataSizeGb) { + metadataSize = GbToKb(maxMetadataSizeGb) + } + return metadataSize +} + +// CreateTP creates LVM Thin Pool +func CreateTP(vgname, tpname string, tpsize, metasize uint64) error { + // TODO: Chunksize adjust based on RAID/JBOD + return exec.Command("lvcreate", + "--thin", vgname+"/"+tpname, + "--size", fmt.Sprintf("%dK", tpsize), + "--poolmetadatasize", fmt.Sprintf("%dK", metasize), + "-c", chunkSize, + "--zero", "n", + ).Run() +} + +// CreateLV creates LVM Logical Volume +func CreateLV(vgname, tpname, lvname string, lvsize uint64) error { + return exec.Command("lvcreate", + "--virtualsize", fmt.Sprintf("%dK", lvsize), + "--thin", + "--name", lvname, + vgname+"/"+tpname, + ).Run() +} + +// MakeXfs creates XFS filesystem +func MakeXfs(dev string) error { + // TODO: Adjust -d su=<>,sw=<> based on RAID/JBOD + return exec.Command("mkfs.xfs", + "-i", "size=512", + "-n", "size=8192", + dev, + ).Run() +} + +// BrickMount mounts the brick LV +func BrickMount(dev, mountdir string) error { + return exec.Command("mount", + "-o", "rw,inode64,noatime,nouuid", + dev, + mountdir, + ).Run() +} + +// BrickUnmount unmounts the Brick +func BrickUnmount(mountdir string) error { + return exec.Command("umount", mountdir).Run() +} + +// RemoveLV removes Logical Volume +func RemoveLV(vgName, lvName string) error { + return exec.Command("lvremove", "-f", vgName+"/"+lvName).Run() +} diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 9c7004e58..7cbdee967 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -10,6 +10,7 @@ import ( "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/pkg/errors" deviceapi "github.com/gluster/glusterd2/plugins/device/api" + "github.com/gluster/glusterd2/plugins/device/deviceutils" "github.com/gorilla/mux" "github.com/pborman/uuid" @@ -62,7 +63,7 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - if checkIfDeviceExist(req.Device, devices) { + if deviceutils.DeviceExist(req.Device, devices) { logger.WithError(err).WithField("device", req.Device).Error("Device already exists") restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Device already exists") return diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index 438882316..7d90cf534 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -40,12 +40,18 @@ func txnPrepareDevice(c transaction.TxnCtx) error { } c.Logger().WithField("device", device).Info("Device setup successful, setting device status to 'Enabled'") + availableSize, extentSize, err := deviceutils.GetVgAvailableSize(vgName) + if err != nil { + return err + } deviceInfo = deviceapi.Info{ - Name: device, - State: deviceapi.DeviceEnabled, + Name: device, + State: deviceapi.DeviceEnabled, + AvailableSize: availableSize, + ExtentSize: extentSize, } - err = addDevice(deviceInfo, peerID) + err = deviceutils.AddDevice(deviceInfo, peerID) if err != nil { c.Logger().WithError(err).WithField("peerid", peerID).Error("Couldn't add deviceinfo to store") return err From b49dc1c0807a211f816f9754c190091b0cf4b9b1 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Thu, 24 May 2018 20:20:17 +0530 Subject: [PATCH 35/54] Revert "Use the built-in "context" package" This reverts commit 5c0eaeaabe517f22276f2b03c2a1ed77e189f4ae. --- glusterd2/commands/peers/peer-rpc-svc.go | 3 +-- glusterd2/transaction/rpc-client.go | 4 ++-- glusterd2/transaction/rpc-service.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/glusterd2/commands/peers/peer-rpc-svc.go b/glusterd2/commands/peers/peer-rpc-svc.go index 21ce49739..dc6c03a5b 100644 --- a/glusterd2/commands/peers/peer-rpc-svc.go +++ b/glusterd2/commands/peers/peer-rpc-svc.go @@ -1,8 +1,6 @@ package peercommands import ( - "context" - "github.com/gluster/glusterd2/glusterd2/events" "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/peer" @@ -13,6 +11,7 @@ import ( "github.com/pborman/uuid" log "github.com/sirupsen/logrus" + "golang.org/x/net/context" "google.golang.org/grpc" ) diff --git a/glusterd2/transaction/rpc-client.go b/glusterd2/transaction/rpc-client.go index fcae57c83..fa5546fcb 100644 --- a/glusterd2/transaction/rpc-client.go +++ b/glusterd2/transaction/rpc-client.go @@ -1,7 +1,6 @@ package transaction import ( - "context" "encoding/json" "errors" @@ -10,6 +9,7 @@ import ( "github.com/pborman/uuid" log "github.com/sirupsen/logrus" + netctx "golang.org/x/net/context" "google.golang.org/grpc" ) @@ -65,7 +65,7 @@ func runStepOn(step string, node uuid.UUID, c TxnCtx) error { var rsp *TxnStepResp - rsp, err = client.RunStep(context.TODO(), req) + rsp, err = client.RunStep(netctx.TODO(), req) if err != nil { logger.WithFields(log.Fields{ "error": err, diff --git a/glusterd2/transaction/rpc-service.go b/glusterd2/transaction/rpc-service.go index 2edc2aa4e..2ccb9982c 100644 --- a/glusterd2/transaction/rpc-service.go +++ b/glusterd2/transaction/rpc-service.go @@ -1,13 +1,13 @@ package transaction import ( - "context" "encoding/json" "errors" "github.com/gluster/glusterd2/glusterd2/servers/peerrpc" log "github.com/sirupsen/logrus" + "golang.org/x/net/context" "google.golang.org/grpc" ) From a84264a067d7ef4247a1f544759748a6b92d598c Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Thu, 24 May 2018 10:51:35 -0400 Subject: [PATCH 36/54] removing list all devices --- pkg/restclient/device.go | 2 +- plugins/device/rest.go | 24 ------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/pkg/restclient/device.go b/pkg/restclient/device.go index 9f7020bc5..c6ace7b12 100644 --- a/pkg/restclient/device.go +++ b/pkg/restclient/device.go @@ -12,6 +12,6 @@ func (c *Client) DeviceAdd(peerid string, device string) (deviceapi.AddDeviceRes req := deviceapi.AddDeviceReq{ Device: device, } - err := c.post("/v1/peers/"+peerid+"/devices", req, http.StatusOK, &peerinfo) + err := c.post("/v1/devices/"+peerid, req, http.StatusOK, &peerinfo) return peerinfo, err } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index f35f7af4d..62aaf6b46 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -131,27 +131,3 @@ func deviceListHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPResponse(ctx, w, http.StatusOK, devices) } - -func listAllHandler(w http.ResponseWriter, r *http.Request) { - - ctx := r.Context() - logger := gdctx.GetReqLogger(ctx) - peers, err := GetPeers() - if err != nil { - logger.WithError(err).Error("Failed to get peers from store") - } - - devices, err := GetDevices(peerID) - if err != nil { - logger.WithError(err).WithField("peerid", peerID).Error("Failed to get devices for peer") - if err == errors.ErrPeerNotFound { - restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) - } else { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - } - return - } - - restutils.SendHTTPResponse(ctx, w, http.StatusOK, devices) - -} From 893c0b1c6d04f92b68b67eed53895d58f7505698 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Fri, 25 May 2018 12:03:14 +0530 Subject: [PATCH 37/54] Set request logger to logrus.FieldLogger interface type Request logger was eariler being set to logrus.Entry type. With this change, request logger is set to logrus.FieldLogger which is an interface and logrus.Entry implements this interface. Further, renamed variables to make difference between logger and entry more clear. This change also fixes a crash in transaction.CreateLockFuncs() which occurs when logger is nil. Logger will be nil if context passed is not dervided from the REST request. Signed-off-by: Prashanth Pai --- glusterd2/gdctx/context.go | 6 +++--- glusterd2/middleware/request_id.go | 4 ++-- glusterd2/middleware/request_logging.go | 6 +++--- glusterd2/transaction/lock.go | 7 +++++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/glusterd2/gdctx/context.go b/glusterd2/gdctx/context.go index cbc248fd2..c10bd9947 100644 --- a/glusterd2/gdctx/context.go +++ b/glusterd2/gdctx/context.go @@ -29,13 +29,13 @@ func GetReqID(ctx context.Context) uuid.UUID { } // WithReqLogger returns a new context with provided logger set as a value in the context. -func WithReqLogger(ctx context.Context, logger *log.Entry) context.Context { +func WithReqLogger(ctx context.Context, logger log.FieldLogger) context.Context { return context.WithValue(ctx, reqLoggerKey, logger) } // GetReqLogger returns logger stored in the context provided. -func GetReqLogger(ctx context.Context) *log.Entry { - reqLogger, ok := ctx.Value(reqLoggerKey).(*log.Entry) +func GetReqLogger(ctx context.Context) log.FieldLogger { + reqLogger, ok := ctx.Value(reqLoggerKey).(log.FieldLogger) if !ok { return nil } diff --git a/glusterd2/middleware/request_id.go b/glusterd2/middleware/request_id.go index 413ca4fea..f2709b3a2 100644 --- a/glusterd2/middleware/request_id.go +++ b/glusterd2/middleware/request_id.go @@ -21,8 +21,8 @@ func ReqIDGenerator(next http.Handler) http.Handler { w.Header().Set("X-Request-ID", reqID.String()) // Create request-scoped logger and set in request context - reqLogger := log.WithField("reqid", reqID.String()) - ctx = gdctx.WithReqLogger(ctx, reqLogger) + reqLoggerEntry := log.WithField("reqid", reqID.String()) + ctx = gdctx.WithReqLogger(ctx, reqLoggerEntry) next.ServeHTTP(w, r.WithContext(ctx)) }) diff --git a/glusterd2/middleware/request_logging.go b/glusterd2/middleware/request_logging.go index ef203614f..cc314d2d7 100644 --- a/glusterd2/middleware/request_logging.go +++ b/glusterd2/middleware/request_logging.go @@ -14,8 +14,8 @@ import ( // Apache Common Log Format (CLF) func LogRequest(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - l := log.WithField("reqid", gdctx.GetReqID(r.Context()).String()) - delete(l.Data, logging.SourceField) - handlers.LoggingHandler(l.Writer(), next).ServeHTTP(w, r) + entry := log.WithField("reqid", gdctx.GetReqID(r.Context()).String()) + delete(entry.Data, logging.SourceField) + handlers.LoggingHandler(entry.Writer(), next).ServeHTTP(w, r) }) } diff --git a/glusterd2/transaction/lock.go b/glusterd2/transaction/lock.go index e15e9ccd2..52a63d109 100644 --- a/glusterd2/transaction/lock.go +++ b/glusterd2/transaction/lock.go @@ -10,6 +10,7 @@ import ( "github.com/coreos/etcd/clientv3/concurrency" "github.com/pborman/uuid" + log "github.com/sirupsen/logrus" ) const ( @@ -102,6 +103,9 @@ func CreateLockFuncs(key string) (LockUnlockFunc, LockUnlockFunc) { lockFunc := func(ctx context.Context) error { logger := gdctx.GetReqLogger(ctx) + if logger == nil { + logger = log.StandardLogger() + } ctx, cancel := context.WithTimeout(ctx, lockObtainTimeout) defer cancel() @@ -122,6 +126,9 @@ func CreateLockFuncs(key string) (LockUnlockFunc, LockUnlockFunc) { unlockFunc := func(ctx context.Context) error { logger := gdctx.GetReqLogger(ctx) + if logger == nil { + logger = log.StandardLogger() + } logger.WithField("key", key).Debug("attempting to unlock") if err := locker.Unlock(context.Background()); err != nil { From 75505b562ae9261ed08ed7cdd8bf8cb361e718c0 Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Fri, 25 May 2018 13:07:33 +0530 Subject: [PATCH 38/54] cli: Added support for Disperse Volume and fixed issue with pure distribute volume Signed-off-by: Aravinda VK --- glustercli/cmd/common.go | 4 ++ glustercli/cmd/volume-create.go | 47 +++++++++++++++++-- .../commands/volumes/volume-create-txn.go | 5 ++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/glustercli/cmd/common.go b/glustercli/cmd/common.go index 509a921d1..4ee9f425d 100644 --- a/glustercli/cmd/common.go +++ b/glustercli/cmd/common.go @@ -35,6 +35,10 @@ func isNoRouteToHostErr(err error) bool { } func handleGlusterdConnectFailure(msg string, err error, https bool, host string, port int, errcode int) { + if err == nil { + return + } + if isConnectionRefusedErr(err) || isNoSuchHostErr(err) || isNoRouteToHostErr(err) { scheme := "http" if https { diff --git a/glustercli/cmd/volume-create.go b/glustercli/cmd/volume-create.go index bd74113f5..b48755fa3 100644 --- a/glustercli/cmd/volume-create.go +++ b/glustercli/cmd/volume-create.go @@ -70,6 +70,11 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { subvols := []api.SubvolReq{} if flagCreateReplicaCount > 0 { // Replicate Volume Support + + if numBricks%flagCreateReplicaCount != 0 { + failure("Invalid number of bricks specified", nil, 1) + } + numSubvols := numBricks / flagCreateReplicaCount for i := 0; i < numSubvols; i++ { @@ -87,13 +92,45 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { ArbiterCount: flagCreateArbiterCount, }) } + } else if flagCreateDisperseCount > 0 || flagCreateDisperseDataCount > 0 || flagCreateRedundancyCount > 0 { + subvolSize := 0 + if flagCreateDisperseCount > 0 { + subvolSize = flagCreateDisperseCount + } else if flagCreateDisperseDataCount > 0 && flagCreateRedundancyCount > 0 { + subvolSize = flagCreateDisperseDataCount + flagCreateRedundancyCount + } + + if subvolSize == 0 { + failure("Invalid disperse-count/disperse-data/disperse-redundancy count", nil, 1) + } + + if numBricks%subvolSize != 0 { + failure("Invalid number of bricks specified", nil, 1) + } + + numSubvols := numBricks / subvolSize + + for i := 0; i < numSubvols; i++ { + idx := i * subvolSize + + subvols = append(subvols, api.SubvolReq{ + Type: "disperse", + Bricks: bricks[idx : idx+subvolSize], + DisperseCount: flagCreateDisperseCount, + DisperseData: flagCreateDisperseDataCount, + DisperseRedundancy: flagCreateRedundancyCount, + }) + } } else { // Default Distribute Volume - subvols = []api.SubvolReq{ - { - Type: "distribute", - Bricks: bricks, - }, + for _, b := range bricks { + subvols = append( + subvols, + api.SubvolReq{ + Type: "distribute", + Bricks: []api.BrickReq{b}, + }, + ) } } diff --git a/glusterd2/commands/volumes/volume-create-txn.go b/glusterd2/commands/volumes/volume-create-txn.go index 301861cc4..351441f0b 100644 --- a/glusterd2/commands/volumes/volume-create-txn.go +++ b/glusterd2/commands/volumes/volume-create-txn.go @@ -51,6 +51,11 @@ func voltypeFromSubvols(req *api.VolCreateReq) volume.VolType { return volume.DistReplicate } return volume.Replicate + case "disperse": + if numSubvols > 1 { + return volume.DistDisperse + } + return volume.Disperse case "distribute": return volume.Distribute default: From d6f862bdd57738f6ca4f2a55980a0a2dd9fd6fe8 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 24 May 2018 16:18:44 +0530 Subject: [PATCH 39/54] Filter peers on basis of metadata either by metadata key/value/key+value Signed-off-by: Vishal Pandey --- e2e/peer_ops_test.go | 52 ++++++++++++++++++++++-- glustercli/cmd/peer.go | 27 +++++++++++-- glusterd2/commands/peers/getpeers.go | 12 +++++- glusterd2/peer/store-utils.go | 59 ++++++++++++++++++++++++++-- pkg/api/peer_req_resp.go | 8 ++++ pkg/restclient/peer.go | 15 ++++--- 6 files changed, 152 insertions(+), 21 deletions(-) diff --git a/e2e/peer_ops_test.go b/e2e/peer_ops_test.go index 14d35897e..444db21bd 100644 --- a/e2e/peer_ops_test.go +++ b/e2e/peer_ops_test.go @@ -4,6 +4,8 @@ import ( "testing" "time" + "github.com/gluster/glusterd2/pkg/api" + "github.com/stretchr/testify/require" ) @@ -26,14 +28,23 @@ func TestAddRemovePeer(t *testing.T) { r.True(g3.IsRunning()) client := initRestclient(g1.ClientAddress) - - _, err2 := client.PeerAdd(g2.PeerAddress) + peerAddReq := api.PeerAddReq{ + Addresses: []string{g2.PeerAddress}, + Metadata: map[string]string { + "owner": "gd2test", + }, + } + _, err2 := client.PeerAdd(peerAddReq) r.Nil(err2) time.Sleep(6 * time.Second) // add peer: ask g1 to add g3 as peer - _, err3 := client.PeerAdd(g3.PeerAddress) + peerAddReq = api.PeerAddReq{ + Addresses: []string{g3.PeerAddress}, + } + + _, err3 := client.PeerAdd(peerAddReq) r.Nil(err3) time.Sleep(6 * time.Second) @@ -43,6 +54,41 @@ func TestAddRemovePeer(t *testing.T) { r.Nil(err4) r.Len(peers, 3) + var matchingQueries []map[string]string + var nonMatchingQueries []map[string]string + + matchingQueries = append(matchingQueries, map[string]string{ + "key": "owner", + "value": "gd2test", + }) + matchingQueries = append(matchingQueries, map[string]string{ + "key": "owner", + }) + matchingQueries = append(matchingQueries, map[string]string{ + "value": "gd2test", + }) + for _, filter := range matchingQueries { + peers, err := client.Peers(filter) + r.Nil(err) + r.Len(peers, 1) + } + + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "key": "owner", + "value": "gd2-test", + }) + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "key": "owners", + }) + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "value": "gd2tests", + }) + for _, filter := range nonMatchingQueries { + peers, err := client.Peers(filter) + r.Nil(err) + r.Len(peers, 0) + } + // remove peer: ask g1 to remove g2 as peer err5 := client.PeerRemove(g2.PeerID()) r.Nil(err5) diff --git a/glustercli/cmd/peer.go b/glustercli/cmd/peer.go index 2d5d64c00..53a5de9fb 100644 --- a/glustercli/cmd/peer.go +++ b/glustercli/cmd/peer.go @@ -6,6 +6,8 @@ import ( "os" "strings" + "github.com/gluster/glusterd2/pkg/api" + "github.com/olekukonko/tablewriter" "github.com/pborman/uuid" log "github.com/sirupsen/logrus" @@ -34,9 +36,11 @@ func init() { peerCmd.AddCommand(peerStatusCmd) - peerCmd.AddCommand(peerListCmd) + peerListCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter by metadata key") + peerListCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter by metadata value") + peerCmd.AddCommand(peerListCmd) - RootCmd.AddCommand(peerCmd) + RootCmd.AddCommand(peerCmd) } var peerCmd = &cobra.Command{ @@ -50,7 +54,10 @@ var peerAddCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { hostname := cmd.Flags().Args()[0] - peer, err := client.PeerAdd(hostname) + peerAddReq := api.PeerAddReq{ + Addresses: []string{hostname}, + } + peer, err := client.PeerAdd(peerAddReq) if err != nil { if verbose { log.WithFields(log.Fields{ @@ -92,7 +99,19 @@ var peerRemoveCmd = &cobra.Command{ } func peerStatusHandler(cmd *cobra.Command) { - peers, err := client.Peers() + var peers api.PeerListResp + var err error + if flagCmdFilterKey == "" && flagCmdFilterValue == "" { + peers, err = client.Peers() + } else if flagCmdFilterKey != "" && flagCmdFilterValue == "" { + peers, err = client.Peers(map[string]string{"key": flagCmdFilterKey}) + } else if flagCmdFilterKey == "" && flagCmdFilterValue != "" { + peers, err = client.Peers(map[string]string{"value": flagCmdFilterValue}) + } else if flagCmdFilterKey != "" && flagCmdFilterValue != "" { + peers, err = client.Peers(map[string]string{"key": flagCmdFilterKey, + "value": flagCmdFilterValue, + }) + } if err != nil { if verbose { log.WithFields(log.Fields{ diff --git a/glusterd2/commands/peers/getpeers.go b/glusterd2/commands/peers/getpeers.go index 19ffa5163..a844b885f 100644 --- a/glusterd2/commands/peers/getpeers.go +++ b/glusterd2/commands/peers/getpeers.go @@ -12,8 +12,16 @@ import ( func getPeersHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - - peers, err := peer.GetPeersF() + keys, keyFound := r.URL.Query()["key"] + values, valueFound := r.URL.Query()["value"] + filterParams := make(map[string]string) + if keyFound { + filterParams["key"] = keys[0] + } + if valueFound { + filterParams["value"] = values[0] + } + peers, err := peer.GetPeersF(filterParams) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) } diff --git a/glusterd2/peer/store-utils.go b/glusterd2/peer/store-utils.go index 8696806b3..9144f1ec4 100644 --- a/glusterd2/peer/store-utils.go +++ b/glusterd2/peer/store-utils.go @@ -20,6 +20,17 @@ const ( peerPrefix string = "peers/" ) +// metadataFilter is a filter type +type metadataFilter uint32 + +// GetPeers Filter Types +const ( + noKeyAndValue metadataFilter = iota + onlyKey + onlyValue + keyAndValue +) + var ( // GetPeerF returns specified peer from the store GetPeerF = GetPeer @@ -87,15 +98,35 @@ func GetInitialCluster() (string, error) { return initialCluster, nil } +// getFilterType return the filter type for peer list +func getFilterType(filterParams map[string]string) metadataFilter { + _, key := filterParams["key"] + _, value := filterParams["value"] + if key && !value { + return onlyKey + } else if value && !key { + return onlyValue + } else if value && key { + return keyAndValue + } + return noKeyAndValue +} + // GetPeers returns all available peers in the store -func GetPeers() ([]*Peer, error) { +func GetPeers(filterParams ...map[string]string) ([]*Peer, error) { resp, err := store.Store.Get(context.TODO(), peerPrefix, clientv3.WithPrefix()) if err != nil { return nil, err } + var filterType metadataFilter + if len(filterParams) == 0 { + filterType = noKeyAndValue + } else { + filterType = getFilterType(filterParams[0]) + } // There will be at least one peer (current node) - peers := make([]*Peer, len(resp.Kvs)) - for i, kv := range resp.Kvs { + var peers []*Peer + for _, kv := range resp.Kvs { var p Peer if err := json.Unmarshal(kv.Value, &p); err != nil { @@ -105,7 +136,27 @@ func GetPeers() ([]*Peer, error) { }).Error("Failed to unmarshal peer") continue } - peers[i] = &p + switch filterType { + + case onlyKey: + if _, keyFound := p.Metadata[filterParams[0]["key"]]; keyFound { + peers = append(peers, &p) + } + case onlyValue: + for _, value := range p.Metadata { + if value == filterParams[0]["value"] { + peers = append(peers, &p) + } + } + case keyAndValue: + if value, keyFound := p.Metadata[filterParams[0]["key"]]; keyFound { + if value == filterParams[0]["value"] { + peers = append(peers, &p) + } + } + default: + peers = append(peers, &p) + } } return peers, nil diff --git a/pkg/api/peer_req_resp.go b/pkg/api/peer_req_resp.go index f1de51a27..83bf1d635 100644 --- a/pkg/api/peer_req_resp.go +++ b/pkg/api/peer_req_resp.go @@ -35,4 +35,12 @@ type PeerEditResp Peer type PeerGetResp Peer // PeerListResp is the response sent for a peer list request +/* +The client can request to filter peer listing based on metadata key/value using query parameters. +Example: + - GET http://localhost:24007/v1/peers?key={keyname}&value={value} + - GET http://localhost:24007/v1/peers?key={keyname} + - GET http://localhost:24007/v1/peers?value={value} +Note - Cannot use query parameters if peerid is also supplied. +*/ type PeerListResp []PeerGetResp diff --git a/pkg/restclient/peer.go b/pkg/restclient/peer.go index 2cf301780..5e2b5b329 100644 --- a/pkg/restclient/peer.go +++ b/pkg/restclient/peer.go @@ -8,12 +8,7 @@ import ( ) // PeerAdd adds a peer to the Cluster -func (c *Client) PeerAdd(host string) (api.PeerAddResp, error) { - - peerAddReq := api.PeerAddReq{ - Addresses: []string{host}, - } - +func (c *Client) PeerAdd(peerAddReq api.PeerAddReq) (api.PeerAddResp, error) { var resp api.PeerAddResp err := c.post("/v1/peers", peerAddReq, http.StatusCreated, &resp) return resp, err @@ -26,8 +21,12 @@ func (c *Client) PeerRemove(peerid string) error { } // Peers gets list of Gluster Peers -func (c *Client) Peers() (api.PeerListResp, error) { +func (c *Client) Peers(filterParams ...map[string]string) (api.PeerListResp, error) { var peers api.PeerListResp - err := c.get("/v1/peers", nil, http.StatusOK, &peers) + var queryString string + if len(filterParams) != 0 { + queryString = getQueryString(filterParams[0]) + } + err := c.get("/v1/peers" + queryString, nil, http.StatusOK, &peers) return peers, err } From 1eac111afefbe810cddb1a99e23a043fd9e877ac Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Fri, 25 May 2018 14:49:47 +0200 Subject: [PATCH 40/54] options: fix typos in profile description for EC volumes Eurasure --> erasure Signed-off-by: Michael Adam --- glusterd2/commands/volumes/grouped-options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glusterd2/commands/volumes/grouped-options.go b/glusterd2/commands/volumes/grouped-options.go index 273af5a4d..f14a3088e 100644 --- a/glusterd2/commands/volumes/grouped-options.go +++ b/glusterd2/commands/volumes/grouped-options.go @@ -79,7 +79,7 @@ var defaultGroupOptions = map[string]*api.OptionGroup{ {Name: "performance.aggregate-size", OnValue: "1MB"}, {Name: "client.event-threads", OnValue: "4"}, {Name: "server.event-threads", OnValue: "4"}}, - "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with SMB access of a Eurasure coded volume"}, + "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with SMB access of a erasure coded volume"}, "profile.FUSE-large-file-EC": {"profile.FUSE-large-file-EC", []api.VolumeOption{{Name: "features.cache-invalidation", OnValue: "on"}, {Name: "features.cache-invalidation-timeout", OnValue: "600"}, @@ -97,7 +97,7 @@ var defaultGroupOptions = map[string]*api.OptionGroup{ {Name: "performance.aggregate-size", OnValue: "1MB"}, {Name: "client.event-threads", OnValue: "4"}, {Name: "server.event-threads", OnValue: "4"}}, - "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with native FUSE mount access of a Eurasure coded volume"}, + "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with native FUSE mount access of a erasure coded volume"}, "profile.nl-cache": {"profile.nl-cache", []api.VolumeOption{{Name: "features.cache-invalidation", OnValue: "on"}, {Name: "features.cache-invalidation-timeout", OnValue: "600"}, From fe14094218d77a7008a43720262df77c21914e82 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Fri, 25 May 2018 14:52:53 +0200 Subject: [PATCH 41/54] options: further improve wording for EC profile Signed-off-by: Michael Adam --- glusterd2/commands/volumes/grouped-options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glusterd2/commands/volumes/grouped-options.go b/glusterd2/commands/volumes/grouped-options.go index f14a3088e..e71759acb 100644 --- a/glusterd2/commands/volumes/grouped-options.go +++ b/glusterd2/commands/volumes/grouped-options.go @@ -79,7 +79,7 @@ var defaultGroupOptions = map[string]*api.OptionGroup{ {Name: "performance.aggregate-size", OnValue: "1MB"}, {Name: "client.event-threads", OnValue: "4"}, {Name: "server.event-threads", OnValue: "4"}}, - "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with SMB access of a erasure coded volume"}, + "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with SMB access to an erasure coded volume"}, "profile.FUSE-large-file-EC": {"profile.FUSE-large-file-EC", []api.VolumeOption{{Name: "features.cache-invalidation", OnValue: "on"}, {Name: "features.cache-invalidation-timeout", OnValue: "600"}, @@ -97,7 +97,7 @@ var defaultGroupOptions = map[string]*api.OptionGroup{ {Name: "performance.aggregate-size", OnValue: "1MB"}, {Name: "client.event-threads", OnValue: "4"}, {Name: "server.event-threads", OnValue: "4"}}, - "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with native FUSE mount access of a erasure coded volume"}, + "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with native FUSE mount access to an erasure coded volume"}, "profile.nl-cache": {"profile.nl-cache", []api.VolumeOption{{Name: "features.cache-invalidation", OnValue: "on"}, {Name: "features.cache-invalidation-timeout", OnValue: "600"}, From 0130bc4f6f7eb2395d1048621bb76dcfbcfc7ad3 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Fri, 25 May 2018 15:17:39 +0200 Subject: [PATCH 42/54] options: fix typo in description of profile.virt Found by Humble Chirammal. Signed-off-by: Michael Adam --- glusterd2/commands/volumes/grouped-options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glusterd2/commands/volumes/grouped-options.go b/glusterd2/commands/volumes/grouped-options.go index e71759acb..1928f5698 100644 --- a/glusterd2/commands/volumes/grouped-options.go +++ b/glusterd2/commands/volumes/grouped-options.go @@ -128,7 +128,7 @@ var defaultGroupOptions = map[string]*api.OptionGroup{ {Name: "cluster.shd-wait-qlength", OnValue: "10000"}, {Name: "features.shard", OnValue: "on"}, {Name: "user.cifs", OnValue: "off"}}, - "Enable this profile, if the Gluster Volume is used to store virtaul machines"}, + "Enable this profile, if the Gluster Volume is used to store virtual machines"}, "profile.test": {"profile.test", []api.VolumeOption{{Name: "afr.eager-lock", OnValue: "on"}, {Name: "gfproxy.afr.eager-lock", OnValue: "on"}}, From 38693af6e085ce938a9b00af6335d12072737eb4 Mon Sep 17 00:00:00 2001 From: rishubhjain Date: Fri, 25 May 2018 12:20:03 -0400 Subject: [PATCH 43/54] Formating go code --- e2e/peer_ops_test.go | 20 ++++++++++---------- glustercli/cmd/peer.go | 28 ++++++++++++++-------------- glusterd2/commands/peers/getpeers.go | 16 ++++++++-------- glusterd2/peer/store-utils.go | 26 +++++++++++++------------- pkg/restclient/peer.go | 10 +++++----- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/e2e/peer_ops_test.go b/e2e/peer_ops_test.go index 444db21bd..8f05d0f7f 100644 --- a/e2e/peer_ops_test.go +++ b/e2e/peer_ops_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/gluster/glusterd2/pkg/api" + "github.com/gluster/glusterd2/pkg/api" "github.com/stretchr/testify/require" ) @@ -28,21 +28,21 @@ func TestAddRemovePeer(t *testing.T) { r.True(g3.IsRunning()) client := initRestclient(g1.ClientAddress) - peerAddReq := api.PeerAddReq{ - Addresses: []string{g2.PeerAddress}, - Metadata: map[string]string { - "owner": "gd2test", - }, - } + peerAddReq := api.PeerAddReq{ + Addresses: []string{g2.PeerAddress}, + Metadata: map[string]string{ + "owner": "gd2test", + }, + } _, err2 := client.PeerAdd(peerAddReq) r.Nil(err2) time.Sleep(6 * time.Second) // add peer: ask g1 to add g3 as peer - peerAddReq = api.PeerAddReq{ - Addresses: []string{g3.PeerAddress}, - } + peerAddReq = api.PeerAddReq{ + Addresses: []string{g3.PeerAddress}, + } _, err3 := client.PeerAdd(peerAddReq) r.Nil(err3) diff --git a/glustercli/cmd/peer.go b/glustercli/cmd/peer.go index 53a5de9fb..ce09d006d 100644 --- a/glustercli/cmd/peer.go +++ b/glustercli/cmd/peer.go @@ -6,7 +6,7 @@ import ( "os" "strings" - "github.com/gluster/glusterd2/pkg/api" + "github.com/gluster/glusterd2/pkg/api" "github.com/olekukonko/tablewriter" "github.com/pborman/uuid" @@ -36,11 +36,11 @@ func init() { peerCmd.AddCommand(peerStatusCmd) - peerListCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter by metadata key") - peerListCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter by metadata value") - peerCmd.AddCommand(peerListCmd) + peerListCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter by metadata key") + peerListCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter by metadata value") + peerCmd.AddCommand(peerListCmd) - RootCmd.AddCommand(peerCmd) + RootCmd.AddCommand(peerCmd) } var peerCmd = &cobra.Command{ @@ -54,9 +54,9 @@ var peerAddCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { hostname := cmd.Flags().Args()[0] - peerAddReq := api.PeerAddReq{ - Addresses: []string{hostname}, - } + peerAddReq := api.PeerAddReq{ + Addresses: []string{hostname}, + } peer, err := client.PeerAdd(peerAddReq) if err != nil { if verbose { @@ -101,15 +101,15 @@ var peerRemoveCmd = &cobra.Command{ func peerStatusHandler(cmd *cobra.Command) { var peers api.PeerListResp var err error - if flagCmdFilterKey == "" && flagCmdFilterValue == "" { - peers, err = client.Peers() + if flagCmdFilterKey == "" && flagCmdFilterValue == "" { + peers, err = client.Peers() } else if flagCmdFilterKey != "" && flagCmdFilterValue == "" { - peers, err = client.Peers(map[string]string{"key": flagCmdFilterKey}) + peers, err = client.Peers(map[string]string{"key": flagCmdFilterKey}) } else if flagCmdFilterKey == "" && flagCmdFilterValue != "" { - peers, err = client.Peers(map[string]string{"value": flagCmdFilterValue}) + peers, err = client.Peers(map[string]string{"value": flagCmdFilterValue}) } else if flagCmdFilterKey != "" && flagCmdFilterValue != "" { - peers, err = client.Peers(map[string]string{"key": flagCmdFilterKey, - "value": flagCmdFilterValue, + peers, err = client.Peers(map[string]string{"key": flagCmdFilterKey, + "value": flagCmdFilterValue, }) } if err != nil { diff --git a/glusterd2/commands/peers/getpeers.go b/glusterd2/commands/peers/getpeers.go index a844b885f..cdc8ce7fe 100644 --- a/glusterd2/commands/peers/getpeers.go +++ b/glusterd2/commands/peers/getpeers.go @@ -12,15 +12,15 @@ import ( func getPeersHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - keys, keyFound := r.URL.Query()["key"] + keys, keyFound := r.URL.Query()["key"] values, valueFound := r.URL.Query()["value"] - filterParams := make(map[string]string) - if keyFound { - filterParams["key"] = keys[0] - } - if valueFound { - filterParams["value"] = values[0] - } + filterParams := make(map[string]string) + if keyFound { + filterParams["key"] = keys[0] + } + if valueFound { + filterParams["value"] = values[0] + } peers, err := peer.GetPeersF(filterParams) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) diff --git a/glusterd2/peer/store-utils.go b/glusterd2/peer/store-utils.go index 9144f1ec4..2cfa1c2a0 100644 --- a/glusterd2/peer/store-utils.go +++ b/glusterd2/peer/store-utils.go @@ -118,12 +118,12 @@ func GetPeers(filterParams ...map[string]string) ([]*Peer, error) { if err != nil { return nil, err } - var filterType metadataFilter - if len(filterParams) == 0 { - filterType = noKeyAndValue - } else { - filterType = getFilterType(filterParams[0]) - } + var filterType metadataFilter + if len(filterParams) == 0 { + filterType = noKeyAndValue + } else { + filterType = getFilterType(filterParams[0]) + } // There will be at least one peer (current node) var peers []*Peer for _, kv := range resp.Kvs { @@ -136,13 +136,13 @@ func GetPeers(filterParams ...map[string]string) ([]*Peer, error) { }).Error("Failed to unmarshal peer") continue } - switch filterType { + switch filterType { - case onlyKey: - if _, keyFound := p.Metadata[filterParams[0]["key"]]; keyFound { - peers = append(peers, &p) - } - case onlyValue: + case onlyKey: + if _, keyFound := p.Metadata[filterParams[0]["key"]]; keyFound { + peers = append(peers, &p) + } + case onlyValue: for _, value := range p.Metadata { if value == filterParams[0]["value"] { peers = append(peers, &p) @@ -156,7 +156,7 @@ func GetPeers(filterParams ...map[string]string) ([]*Peer, error) { } default: peers = append(peers, &p) - } + } } return peers, nil diff --git a/pkg/restclient/peer.go b/pkg/restclient/peer.go index 5e2b5b329..126fd8f99 100644 --- a/pkg/restclient/peer.go +++ b/pkg/restclient/peer.go @@ -23,10 +23,10 @@ func (c *Client) PeerRemove(peerid string) error { // Peers gets list of Gluster Peers func (c *Client) Peers(filterParams ...map[string]string) (api.PeerListResp, error) { var peers api.PeerListResp - var queryString string - if len(filterParams) != 0 { - queryString = getQueryString(filterParams[0]) - } - err := c.get("/v1/peers" + queryString, nil, http.StatusOK, &peers) + var queryString string + if len(filterParams) != 0 { + queryString = getQueryString(filterParams[0]) + } + err := c.get("/v1/peers"+queryString, nil, http.StatusOK, &peers) return peers, err } From 0a39d566d161aa654f7d8c06cbdb3834a4fa7bfe Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 28 May 2018 12:25:31 +0530 Subject: [PATCH 44/54] set defaultpeerport as part of setting default configuration Signed-off-by: Madhu Rajanna --- glusterd2/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glusterd2/config.go b/glusterd2/config.go index fa4402f9c..33b26f5b8 100644 --- a/glusterd2/config.go +++ b/glusterd2/config.go @@ -95,7 +95,9 @@ func setDefaults() error { if port == "" { port = defaultpeerport } + config.Set("peeraddress", host+":"+port) + config.Set("defaultpeerport", defaultpeerport) return nil } From 2ff4f66c584361f5e232d0f83e597a93b9d36cc9 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 28 May 2018 12:39:09 +0530 Subject: [PATCH 45/54] transaction: Add new transaction locking methods --- glusterd2/transaction/lock.go | 58 ++++++++++++++++++++++++++-- glusterd2/transaction/transaction.go | 31 +++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/glusterd2/transaction/lock.go b/glusterd2/transaction/lock.go index 52a63d109..4e321291b 100644 --- a/glusterd2/transaction/lock.go +++ b/glusterd2/transaction/lock.go @@ -18,9 +18,13 @@ const ( lockObtainTimeout = 5 * time.Second ) -// ErrLockTimeout is the error returned when lock could not be obtained -// and the request timed out -var ErrLockTimeout = errors.New("could not obtain lock: another conflicting transaction may be in progress") +var ( + // ErrLockTimeout is the error returned when lock could not be obtained + // and the request timed out + ErrLockTimeout = errors.New("could not obtain lock: another conflicting transaction may be in progress") + // ErrLockExists is returned when a lock already exists within the transaction + ErrLockExists = errors.New("existing lock found for given lock ID") +) // createLockStepFunc returns the registry IDs of StepFuncs which lock/unlock the given key. // If existing StepFuncs are not found, new funcs are created and registered. @@ -142,3 +146,51 @@ func CreateLockFuncs(key string) (LockUnlockFunc, LockUnlockFunc) { return lockFunc, unlockFunc } + +func (t *Txn) lock(lockID string) error { + // Ensure that no prior lock exists for the given lockID in this transaction + if _, ok := t.locks[lockID]; ok { + return ErrLockExists + } + + logger := t.Ctx.Logger().WithField("lockID", lockID) + logger.Debug("attempting to obtain lock") + + key := lockPrefix + lockID + locker := concurrency.NewMutex(store.Store.Session, key) + + ctx, cancel := context.WithTimeout(store.Store.Ctx(), lockObtainTimeout) + defer cancel() + + err := locker.Lock(ctx) + switch err { + case nil: + logger.Debug("lock obtained") + // Attach lock to the transaction + t.locks[lockID] = locker + + case context.DeadlineExceeded: + // Propagate this all the way back to the client as a HTTP 409 response + logger.Debug("timeout: failed to obtain lock") + err = ErrLockTimeout + + default: + logger.WithError(err).Error("failed to obtain lock") + } + + return err +} + +// Lock obtains a cluster wide transaction lock on the given lockID/lockIDs, +// and attaches the obtained locks to the transaction +func (t *Txn) Lock(lockID string, lockIDs ...string) error { + if err := t.lock(lockID); err != nil { + return err + } + for _, id := range lockIDs { + if err := t.lock(id); err != nil { + return err + } + } + return nil +} diff --git a/glusterd2/transaction/transaction.go b/glusterd2/transaction/transaction.go index 7a4350feb..1aa6b4b6f 100644 --- a/glusterd2/transaction/transaction.go +++ b/glusterd2/transaction/transaction.go @@ -10,6 +10,7 @@ import ( "github.com/gluster/glusterd2/glusterd2/store" "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3/concurrency" "github.com/pborman/uuid" log "github.com/sirupsen/logrus" ) @@ -24,6 +25,8 @@ var expTxn = expvar.NewMap("txn") type Txn struct { id uuid.UUID reqID uuid.UUID + locks map[string]*concurrency.Mutex + Ctx TxnCtx Steps []*Step @@ -39,14 +42,32 @@ type Txn struct { // NewTxn returns an initialized Txn without any steps func NewTxn(ctx context.Context) *Txn { t := new(Txn) + t.id = uuid.NewRandom() t.reqID = gdctx.GetReqID(ctx) + t.locks = make(map[string]*concurrency.Mutex) + prefix := txnPrefix + t.id.String() t.Ctx = NewCtxWithLogFields(log.Fields{ "txnid": t.id.String(), "reqid": t.reqID.String(), }).WithPrefix(prefix) + t.Ctx.Logger().Debug("new transaction created") + return t +} + +// NewTxnWithLocks returns an empty Txn with locks obtained on given lockIDs +func NewTxnWithLocks(ctx context.Context, lockIDs ...string) *Txn { + t := NewTxn(ctx) + + for _, id := range lockIDs { + if err := t.Lock(id); err != nil { + t.Done() + return nil + } + } + return t } @@ -56,6 +77,16 @@ func (t *Txn) Cleanup() { expTxn.Add("initiated_txn_in_progress", -1) } +// Done releases any obtained locks and cleans up the transaction namespace +// Done must be called after a transaction ends +func (t *Txn) Done() { + // Release obtained locks + for _, locker := range t.locks { + locker.Unlock(context.Background()) + } + t.Cleanup() +} + func (t *Txn) checkAlive() error { if len(t.Nodes) == 0 { From 0d145ced9c733daa9aea8b9005975323348913c3 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 28 May 2018 12:46:18 +0530 Subject: [PATCH 46/54] transaction: Replace Cleanup with Done --- glusterd2/commands/peers/editpeer.go | 2 +- glusterd2/commands/volumes/bricks-status.go | 2 +- glusterd2/commands/volumes/volume-create.go | 2 +- glusterd2/commands/volumes/volume-delete.go | 2 +- glusterd2/commands/volumes/volume-expand.go | 2 +- glusterd2/commands/volumes/volume-option.go | 2 +- glusterd2/commands/volumes/volume-start.go | 2 +- glusterd2/commands/volumes/volume-statedump.go | 2 +- glusterd2/commands/volumes/volume-stop.go | 2 +- glusterd2/transaction/transaction.go | 11 +++-------- plugins/bitrot/rest.go | 8 ++++---- plugins/device/rest.go | 2 +- plugins/georeplication/rest.go | 16 ++++++++-------- plugins/glustershd/rest.go | 4 ++-- plugins/quota/rest.go | 2 +- 15 files changed, 28 insertions(+), 33 deletions(-) diff --git a/glusterd2/commands/peers/editpeer.go b/glusterd2/commands/peers/editpeer.go index 37203bb02..25263e447 100644 --- a/glusterd2/commands/peers/editpeer.go +++ b/glusterd2/commands/peers/editpeer.go @@ -41,7 +41,7 @@ func editPeer(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() lock, unlock, err := transaction.CreateLockSteps(peerID) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) diff --git a/glusterd2/commands/volumes/bricks-status.go b/glusterd2/commands/volumes/bricks-status.go index 62294670f..2047b7d46 100644 --- a/glusterd2/commands/volumes/bricks-status.go +++ b/glusterd2/commands/volumes/bricks-status.go @@ -80,7 +80,7 @@ func volumeBricksStatusHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Steps = []*transaction.Step{ { DoFunc: "bricks-status.Check", diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index d9955737d..1de0fbb45 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -117,7 +117,7 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() lock, unlock, err := transaction.CreateLockSteps(req.Name) if err != nil { diff --git a/glusterd2/commands/volumes/volume-delete.go b/glusterd2/commands/volumes/volume-delete.go index 06b35f38e..f80792ac9 100644 --- a/glusterd2/commands/volumes/volume-delete.go +++ b/glusterd2/commands/volumes/volume-delete.go @@ -96,7 +96,7 @@ func volumeDeleteHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Steps = []*transaction.Step{ { diff --git a/glusterd2/commands/volumes/volume-expand.go b/glusterd2/commands/volumes/volume-expand.go index e8a190253..91d61d1b2 100644 --- a/glusterd2/commands/volumes/volume-expand.go +++ b/glusterd2/commands/volumes/volume-expand.go @@ -93,7 +93,7 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() nodes, err := req.Nodes() if err != nil { diff --git a/glusterd2/commands/volumes/volume-option.go b/glusterd2/commands/volumes/volume-option.go index 9088869ab..247c01640 100644 --- a/glusterd2/commands/volumes/volume-option.go +++ b/glusterd2/commands/volumes/volume-option.go @@ -182,7 +182,7 @@ func volumeOptionsHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() allNodes, err := peer.GetPeerIDs() if err != nil { diff --git a/glusterd2/commands/volumes/volume-start.go b/glusterd2/commands/volumes/volume-start.go index c13aca2eb..262089e70 100644 --- a/glusterd2/commands/volumes/volume-start.go +++ b/glusterd2/commands/volumes/volume-start.go @@ -107,7 +107,7 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Steps = []*transaction.Step{ { diff --git a/glusterd2/commands/volumes/volume-statedump.go b/glusterd2/commands/volumes/volume-statedump.go index c32cea0e3..84ed76c0d 100644 --- a/glusterd2/commands/volumes/volume-statedump.go +++ b/glusterd2/commands/volumes/volume-statedump.go @@ -130,7 +130,7 @@ func volumeStatedumpHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Steps = []*transaction.Step{ { diff --git a/glusterd2/commands/volumes/volume-stop.go b/glusterd2/commands/volumes/volume-stop.go index 4809209c7..221a9932f 100644 --- a/glusterd2/commands/volumes/volume-stop.go +++ b/glusterd2/commands/volumes/volume-stop.go @@ -105,7 +105,7 @@ func volumeStopHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Steps = []*transaction.Step{ { diff --git a/glusterd2/transaction/transaction.go b/glusterd2/transaction/transaction.go index 1aa6b4b6f..4b011b5b3 100644 --- a/glusterd2/transaction/transaction.go +++ b/glusterd2/transaction/transaction.go @@ -54,6 +54,7 @@ func NewTxn(ctx context.Context) *Txn { }).WithPrefix(prefix) t.Ctx.Logger().Debug("new transaction created") + expTxn.Add("initiated_txn_in_progress", 1) return t } @@ -71,12 +72,6 @@ func NewTxnWithLocks(ctx context.Context, lockIDs ...string) *Txn { return t } -// Cleanup cleans the leftovers after a transaction ends -func (t *Txn) Cleanup() { - store.Store.Delete(context.TODO(), t.Ctx.Prefix(), clientv3.WithPrefix()) - expTxn.Add("initiated_txn_in_progress", -1) -} - // Done releases any obtained locks and cleans up the transaction namespace // Done must be called after a transaction ends func (t *Txn) Done() { @@ -84,7 +79,8 @@ func (t *Txn) Done() { for _, locker := range t.locks { locker.Unlock(context.Background()) } - t.Cleanup() + store.Store.Delete(context.TODO(), t.Ctx.Prefix(), clientv3.WithPrefix()) + expTxn.Add("initiated_txn_in_progress", -1) } func (t *Txn) checkAlive() error { @@ -115,7 +111,6 @@ func (t *Txn) Do() error { } t.Ctx.Logger().Debug("Starting transaction") - expTxn.Add("initiated_txn_in_progress", 1) for i, s := range t.Steps { if s.Skip { diff --git a/plugins/bitrot/rest.go b/plugins/bitrot/rest.go index 958dbb78d..0b0fffb09 100644 --- a/plugins/bitrot/rest.go +++ b/plugins/bitrot/rest.go @@ -57,7 +57,7 @@ func bitrotEnableHandler(w http.ResponseWriter, r *http.Request) { // Transaction which starts bitd and scrubber on all nodes. txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() if err := txn.Ctx.Set("volinfo", volinfo); err != nil { logger.WithError(err).Error("failed to set volinfo in transaction context") @@ -140,7 +140,7 @@ func bitrotDisableHandler(w http.ResponseWriter, r *http.Request) { // Transaction which stop bitd and scrubber on all nodes. txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() if err := txn.Ctx.Set("volinfo", volinfo); err != nil { logger.WithError(err).Error("failed to set volinfo in transaction context") @@ -221,7 +221,7 @@ func bitrotScrubOndemandHandler(w http.ResponseWriter, r *http.Request) { // Transaction which starts scrubber on demand. txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() //Lock on Volume Name lock, unlock, err := transaction.CreateLockSteps(volName) @@ -288,7 +288,7 @@ func bitrotScrubStatusHandler(w http.ResponseWriter, r *http.Request) { // Transaction which gets scrubber status. txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() //Lock on Volume Name lock, unlock, err := transaction.CreateLockSteps(volName) diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 310d7ed44..d84f28ad0 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -71,7 +71,7 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Nodes = []uuid.UUID{peerInfo.ID} txn.Steps = []*transaction.Step{ diff --git a/plugins/georeplication/rest.go b/plugins/georeplication/rest.go index 18f86fbbd..0b3b6446b 100644 --- a/plugins/georeplication/rest.go +++ b/plugins/georeplication/rest.go @@ -160,7 +160,7 @@ func georepCreateHandler(w http.ResponseWriter, r *http.Request) { // Transaction which updates the Geo-rep session txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() // Lock on Master Volume name lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) @@ -292,7 +292,7 @@ func georepActionHandler(w http.ResponseWriter, r *http.Request, action actionTy } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) if err != nil { @@ -424,7 +424,7 @@ func georepDeleteHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) if err != nil { @@ -514,7 +514,7 @@ func georepStatusHandler(w http.ResponseWriter, r *http.Request) { // Status Transaction txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() txn.Nodes = vol.Nodes() txn.Steps = []*transaction.Step{ @@ -785,7 +785,7 @@ func georepConfigSetHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() // TODO: change the lock key lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) if err != nil { @@ -927,7 +927,7 @@ func georepConfigResetHandler(w http.ResponseWriter, r *http.Request) { } txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() // TODO: change the lock key lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) if err != nil { @@ -1024,7 +1024,7 @@ func georepSSHKeyGenerateHandler(w http.ResponseWriter, r *http.Request) { // Transaction which updates the Geo-rep session txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() // Lock on Master Volume name lock, unlock, err := transaction.CreateLockSteps(volname) @@ -1120,7 +1120,7 @@ func georepSSHKeyPushHandler(w http.ResponseWriter, r *http.Request) { // Transaction which updates the Geo-rep session txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() // Lock on Master Volume name lock, unlock, err := transaction.CreateLockSteps(volname) diff --git a/plugins/glustershd/rest.go b/plugins/glustershd/rest.go index a62f46ed6..5c05deec3 100644 --- a/plugins/glustershd/rest.go +++ b/plugins/glustershd/rest.go @@ -58,7 +58,7 @@ func glustershEnableHandler(w http.ResponseWriter, r *http.Request) { // Transaction which starts self heal daemon on all nodes with atleast one brick. txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() //Lock on Volume Name lock, unlock, err := transaction.CreateLockSteps(volname) @@ -140,7 +140,7 @@ func glustershDisableHandler(w http.ResponseWriter, r *http.Request) { // Transaction which checks if all replicate volumes are stopped before // stopping the self-heal daemon. txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() // Lock on volume name. lock, unlock, err := transaction.CreateLockSteps(volname) diff --git a/plugins/quota/rest.go b/plugins/quota/rest.go index 87d8a4262..f34ae3d45 100644 --- a/plugins/quota/rest.go +++ b/plugins/quota/rest.go @@ -74,7 +74,7 @@ func quotaEnableHandler(w http.ResponseWriter, r *http.Request) { vol.Options[quotaEnabledKey] = "on" txn := transaction.NewTxn(ctx) - defer txn.Cleanup() + defer txn.Done() if err := txn.Ctx.Set("volinfo", vol); err != nil { logger.WithError(err).Error("failed to set volinfo in transaction context") From bc51bf87400a5c90ae32cf8bd2194c271cd63fd9 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 28 May 2018 13:00:18 +0530 Subject: [PATCH 47/54] transaction: Replace use of CreateLockFuncs with NewTxnWithLocks --- glusterd2/commands/volumes/volume-delete.go | 11 +--- glusterd2/commands/volumes/volume-edit.go | 6 +- glusterd2/commands/volumes/volume-option.go | 11 +--- glusterd2/commands/volumes/volume-start.go | 11 +--- .../commands/volumes/volume-statedump.go | 9 +-- glusterd2/commands/volumes/volume-stop.go | 11 +--- glusterd2/transaction/lock.go | 58 ------------------- glusterd2/transaction/transaction.go | 6 +- plugins/device/rest.go | 9 +-- 9 files changed, 24 insertions(+), 108 deletions(-) diff --git a/glusterd2/commands/volumes/volume-delete.go b/glusterd2/commands/volumes/volume-delete.go index f80792ac9..1fac894a5 100644 --- a/glusterd2/commands/volumes/volume-delete.go +++ b/glusterd2/commands/volumes/volume-delete.go @@ -66,10 +66,8 @@ func volumeDeleteHandler(w http.ResponseWriter, r *http.Request) { logger := gdctx.GetReqLogger(ctx) volname := mux.Vars(r)["volname"] - lock, unlock := transaction.CreateLockFuncs(volname) - // Taking a lock outside the txn as volinfo.Nodes() must also - // be populated holding the lock. See issue #510 - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -77,7 +75,7 @@ func volumeDeleteHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() volinfo, err := volume.GetVolume(volname) if err != nil { @@ -95,9 +93,6 @@ func volumeDeleteHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - txn.Steps = []*transaction.Step{ { DoFunc: "vol-delete.DeleteVolfiles", diff --git a/glusterd2/commands/volumes/volume-edit.go b/glusterd2/commands/volumes/volume-edit.go index e8dd0a1a9..faa1af4b2 100644 --- a/glusterd2/commands/volumes/volume-edit.go +++ b/glusterd2/commands/volumes/volume-edit.go @@ -29,8 +29,8 @@ func volumeEditHandler(w http.ResponseWriter, r *http.Request) { } //Lock on Volume Name - lock, unlock := transaction.CreateLockFuncs(volname) - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -38,7 +38,7 @@ func volumeEditHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() //validate volume name volinfo, err := volume.GetVolume(volname) diff --git a/glusterd2/commands/volumes/volume-option.go b/glusterd2/commands/volumes/volume-option.go index 247c01640..b0605c90d 100644 --- a/glusterd2/commands/volumes/volume-option.go +++ b/glusterd2/commands/volumes/volume-option.go @@ -158,10 +158,8 @@ func volumeOptionsHandler(w http.ResponseWriter, r *http.Request) { return } - lock, unlock := transaction.CreateLockFuncs(volname) - // Taking a lock outside the txn as volinfo.Nodes() must also - // be populated holding the lock. - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -169,7 +167,7 @@ func volumeOptionsHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() volinfo, err := volume.GetVolume(volname) if err != nil { @@ -181,9 +179,6 @@ func volumeOptionsHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - allNodes, err := peer.GetPeerIDs() if err != nil { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) diff --git a/glusterd2/commands/volumes/volume-start.go b/glusterd2/commands/volumes/volume-start.go index 262089e70..824980831 100644 --- a/glusterd2/commands/volumes/volume-start.go +++ b/glusterd2/commands/volumes/volume-start.go @@ -78,10 +78,8 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { return } - lock, unlock := transaction.CreateLockFuncs(volname) - // Taking a lock outside the txn as volinfo.Nodes() must also - // be populated holding the lock. See issue #510 - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -89,7 +87,7 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() volinfo, err := volume.GetVolume(volname) if err != nil { @@ -106,9 +104,6 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - txn.Steps = []*transaction.Step{ { DoFunc: "vol-start.StartBricks", diff --git a/glusterd2/commands/volumes/volume-statedump.go b/glusterd2/commands/volumes/volume-statedump.go index 84ed76c0d..6de69a094 100644 --- a/glusterd2/commands/volumes/volume-statedump.go +++ b/glusterd2/commands/volumes/volume-statedump.go @@ -103,8 +103,8 @@ func volumeStatedumpHandler(w http.ResponseWriter, r *http.Request) { return } - lock, unlock := transaction.CreateLockFuncs(volname) - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -112,7 +112,7 @@ func volumeStatedumpHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() volinfo, err := volume.GetVolume(volname) if err != nil { @@ -129,9 +129,6 @@ func volumeStatedumpHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - txn.Steps = []*transaction.Step{ { DoFunc: "vol-statedump.TakeStatedump", diff --git a/glusterd2/commands/volumes/volume-stop.go b/glusterd2/commands/volumes/volume-stop.go index 221a9932f..6238825dc 100644 --- a/glusterd2/commands/volumes/volume-stop.go +++ b/glusterd2/commands/volumes/volume-stop.go @@ -76,10 +76,8 @@ func volumeStopHandler(w http.ResponseWriter, r *http.Request) { logger := gdctx.GetReqLogger(ctx) volname := mux.Vars(r)["volname"] - lock, unlock := transaction.CreateLockFuncs(volname) - // Taking a lock outside the txn as volinfo.Nodes() must also - // be populated holding the lock. See issue #510 - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -87,7 +85,7 @@ func volumeStopHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() volinfo, err := volume.GetVolume(volname) if err != nil { @@ -104,9 +102,6 @@ func volumeStopHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - txn.Steps = []*transaction.Step{ { DoFunc: "vol-stop.StopBricks", diff --git a/glusterd2/transaction/lock.go b/glusterd2/transaction/lock.go index 4e321291b..0c3364014 100644 --- a/glusterd2/transaction/lock.go +++ b/glusterd2/transaction/lock.go @@ -10,7 +10,6 @@ import ( "github.com/coreos/etcd/clientv3/concurrency" "github.com/pborman/uuid" - log "github.com/sirupsen/logrus" ) const ( @@ -90,63 +89,6 @@ func CreateLockSteps(key string) (*Step, *Step, error) { return lockStep, unlockStep, nil } -// LockUnlockFunc is signature of functions used for distributed locking -// and unlocking. -type LockUnlockFunc func(ctx context.Context) error - -// CreateLockFuncs creates and returns functions for distributed lock and -// unlock. This is similar to CreateLockSteps() but returns normal functions. -func CreateLockFuncs(key string) (LockUnlockFunc, LockUnlockFunc) { - - key = lockPrefix + key - locker := concurrency.NewMutex(store.Store.Session, key) - - // TODO: There is an opportunity for refactor here to re-use code - // between CreateLockFunc and CreateLockSteps. This variant doesn't - // have registry either. - - lockFunc := func(ctx context.Context) error { - logger := gdctx.GetReqLogger(ctx) - if logger == nil { - logger = log.StandardLogger() - } - - ctx, cancel := context.WithTimeout(ctx, lockObtainTimeout) - defer cancel() - - logger.WithField("key", key).Debug("attempting to lock") - err := locker.Lock(ctx) - switch err { - case nil: - logger.WithField("key", key).Debug("lock obtained") - case context.DeadlineExceeded: - // Propagate this all the way back to the client as a HTTP 409 response - logger.WithField("key", key).Debug("timeout: failed to obtain lock") - err = ErrLockTimeout - } - - return err - } - - unlockFunc := func(ctx context.Context) error { - logger := gdctx.GetReqLogger(ctx) - if logger == nil { - logger = log.StandardLogger() - } - - logger.WithField("key", key).Debug("attempting to unlock") - if err := locker.Unlock(context.Background()); err != nil { - logger.WithField("key", key).WithError(err).Error("unlock failed") - return err - } - - logger.WithField("key", key).Debug("lock unlocked") - return nil - } - - return lockFunc, unlockFunc -} - func (t *Txn) lock(lockID string) error { // Ensure that no prior lock exists for the given lockID in this transaction if _, ok := t.locks[lockID]; ok { diff --git a/glusterd2/transaction/transaction.go b/glusterd2/transaction/transaction.go index 4b011b5b3..902561393 100644 --- a/glusterd2/transaction/transaction.go +++ b/glusterd2/transaction/transaction.go @@ -59,17 +59,17 @@ func NewTxn(ctx context.Context) *Txn { } // NewTxnWithLocks returns an empty Txn with locks obtained on given lockIDs -func NewTxnWithLocks(ctx context.Context, lockIDs ...string) *Txn { +func NewTxnWithLocks(ctx context.Context, lockIDs ...string) (*Txn, error) { t := NewTxn(ctx) for _, id := range lockIDs { if err := t.Lock(id); err != nil { t.Done() - return nil + return nil, err } } - return t + return t, nil } // Done releases any obtained locks and cleans up the transaction namespace diff --git a/plugins/device/rest.go b/plugins/device/rest.go index d84f28ad0..5a72bb753 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -33,8 +33,8 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { return } - lock, unlock := transaction.CreateLockFuncs(peerID) - if err := lock(ctx); err != nil { + txn, err := transaction.NewTxnWithLocks(ctx, peerID) + if err != nil { if err == transaction.ErrLockTimeout { restutils.SendHTTPError(ctx, w, http.StatusConflict, err) } else { @@ -42,7 +42,7 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { } return } - defer unlock(ctx) + defer txn.Done() peerInfo, err := peer.GetPeer(peerID) if err != nil { @@ -70,9 +70,6 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { } } - txn := transaction.NewTxn(ctx) - defer txn.Done() - txn.Nodes = []uuid.UUID{peerInfo.ID} txn.Steps = []*transaction.Step{ { From b709a45a0bbace64b2fea65c97858421ba3dce1f Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 28 May 2018 15:09:36 +0530 Subject: [PATCH 48/54] transaction: Remove CreateLockSteps --- glusterd2/commands/peers/editpeer.go | 13 +- glusterd2/commands/volumes/volume-create.go | 14 +- glusterd2/commands/volumes/volume-expand.go | 22 +-- glusterd2/commands/volumes/volume-reset.go | 21 +-- glusterd2/transaction/lock.go | 66 -------- plugins/bitrot/rest.go | 98 ++++++------ plugins/georeplication/rest.go | 160 ++++++++++---------- plugins/glustershd/rest.go | 49 +++--- plugins/quota/rest.go | 22 +-- 9 files changed, 191 insertions(+), 274 deletions(-) diff --git a/glusterd2/commands/peers/editpeer.go b/glusterd2/commands/peers/editpeer.go index 25263e447..97c6b488b 100644 --- a/glusterd2/commands/peers/editpeer.go +++ b/glusterd2/commands/peers/editpeer.go @@ -40,21 +40,22 @@ func editPeer(w http.ResponseWriter, r *http.Request) { } } - txn := transaction.NewTxn(ctx) - defer txn.Done() - lock, unlock, err := transaction.CreateLockSteps(peerID) + txn, err := transaction.NewTxnWithLocks(ctx, peerID) if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } return } + defer txn.Done() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "peer-edit", Nodes: []uuid.UUID{gdctx.MyUUID}, }, - unlock, } err = txn.Ctx.Set("peerid", peerID) if err != nil { diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index 1de0fbb45..d3dd6f3e7 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -116,17 +116,18 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - - lock, unlock, err := transaction.CreateLockSteps(req.Name) + txn, err := transaction.NewTxnWithLocks(ctx, req.Name) if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } return } + defer txn.Done() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-create.CreateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -145,7 +146,6 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { UndoFunc: "vol-create.UndoStoreVolume", Nodes: []uuid.UUID{gdctx.MyUUID}, }, - unlock, } if err := txn.Ctx.Set("req", &req); err != nil { diff --git a/glusterd2/commands/volumes/volume-expand.go b/glusterd2/commands/volumes/volume-expand.go index 91d61d1b2..89bd25cd5 100644 --- a/glusterd2/commands/volumes/volume-expand.go +++ b/glusterd2/commands/volumes/volume-expand.go @@ -66,6 +66,17 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { return } + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + volinfo, err := volume.GetVolume(volname) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) @@ -86,15 +97,6 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { } } - lock, unlock, err := transaction.CreateLockSteps(volname) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - - txn := transaction.NewTxn(ctx) - defer txn.Done() - nodes, err := req.Nodes() if err != nil { logger.WithError(err).Error("could not prepare node list") @@ -113,7 +115,6 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { // TODO: This is a lot of steps. We can combine a few if we // do not re-use the same step functions across multiple // volume operations. - lock, { DoFunc: "vol-expand.ValidateAndPrepare", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -140,7 +141,6 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "vol-expand.NotifyClients", Nodes: allNodes, }, - unlock, } if err := txn.Ctx.Set("req", &req); err != nil { diff --git a/glusterd2/commands/volumes/volume-reset.go b/glusterd2/commands/volumes/volume-reset.go index 982bc5fd8..6f89332b6 100644 --- a/glusterd2/commands/volumes/volume-reset.go +++ b/glusterd2/commands/volumes/volume-reset.go @@ -22,6 +22,18 @@ func volumeResetHandler(w http.ResponseWriter, r *http.Request) { logger := gdctx.GetReqLogger(ctx) volname := mux.Vars(r)["volname"] + + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + volinfo, err := volume.GetVolume(volname) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrVolNotFound) @@ -34,7 +46,6 @@ func volumeResetHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) // Delete the option after checking for volopt flags opReset := false for _, k := range req.Options { @@ -75,12 +86,6 @@ func volumeResetHandler(w http.ResponseWriter, r *http.Request) { return } - lock, unlock, err := transaction.CreateLockSteps(volinfo.Name) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - allNodes, err := peer.GetPeerIDs() if err != nil { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) @@ -88,7 +93,6 @@ func volumeResetHandler(w http.ResponseWriter, r *http.Request) { } txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -97,7 +101,6 @@ func volumeResetHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "vol-option.NotifyVolfileChange", Nodes: allNodes, }, - unlock, } if err := txn.Ctx.Set("volinfo", volinfo); err != nil { diff --git a/glusterd2/transaction/lock.go b/glusterd2/transaction/lock.go index 0c3364014..1ed436a66 100644 --- a/glusterd2/transaction/lock.go +++ b/glusterd2/transaction/lock.go @@ -5,11 +5,9 @@ import ( "errors" "time" - "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/store" "github.com/coreos/etcd/clientv3/concurrency" - "github.com/pborman/uuid" ) const ( @@ -25,70 +23,6 @@ var ( ErrLockExists = errors.New("existing lock found for given lock ID") ) -// createLockStepFunc returns the registry IDs of StepFuncs which lock/unlock the given key. -// If existing StepFuncs are not found, new funcs are created and registered. -func createLockStepFunc(key string) (string, string, error) { - lockFuncID := key + ".Lock" - unlockFuncID := key + ".Unlock" - - _, lockFuncFound := getStepFunc(lockFuncID) - _, unlockFuncFound := getStepFunc(unlockFuncID) - - if lockFuncFound && unlockFuncFound { - return lockFuncID, unlockFuncID, nil - } - - key = lockPrefix + key - locker := concurrency.NewMutex(store.Store.Session, key) - - lockFunc := func(c TxnCtx) error { - - ctx, cancel := context.WithTimeout(context.Background(), lockObtainTimeout) - defer cancel() - - c.Logger().WithField("key", key).Debug("attempting to lock") - err := locker.Lock(ctx) - switch err { - case nil: - c.Logger().WithField("key", key).Debug("lock obtained") - case context.DeadlineExceeded: - // Propagate this all the way back to the client as a HTTP 409 response - c.Logger().WithField("key", key).Debug("timeout: failed to obtain lock") - err = ErrLockTimeout - } - - return err - } - RegisterStepFunc(lockFunc, lockFuncID) - - unlockFunc := func(c TxnCtx) error { - - c.Logger().WithField("key", key).Debug("attempting to unlock") - err := locker.Unlock(context.Background()) - if err == nil { - c.Logger().WithField("key", key).Debug("lock unlocked") - } - - return err - } - RegisterStepFunc(unlockFunc, unlockFuncID) - - return lockFuncID, unlockFuncID, nil -} - -// CreateLockSteps returns a lock and an unlock Step which lock/unlock the given key -func CreateLockSteps(key string) (*Step, *Step, error) { - lockFunc, unlockFunc, err := createLockStepFunc(key) - if err != nil { - return nil, nil, err - } - - lockStep := &Step{lockFunc, unlockFunc, []uuid.UUID{gdctx.MyUUID}, false} - unlockStep := &Step{unlockFunc, "", []uuid.UUID{gdctx.MyUUID}, false} - - return lockStep, unlockStep, nil -} - func (t *Txn) lock(lockID string) error { // Ensure that no prior lock exists for the given lockID in this transaction if _, ok := t.locks[lockID]; ok { diff --git a/plugins/bitrot/rest.go b/plugins/bitrot/rest.go index 0b0fffb09..fa1864810 100644 --- a/plugins/bitrot/rest.go +++ b/plugins/bitrot/rest.go @@ -25,6 +25,17 @@ func bitrotEnableHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volName) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Validate volume existence volinfo, err := volume.GetVolume(volName) if err != nil { @@ -55,26 +66,14 @@ func bitrotEnableHandler(w http.ResponseWriter, r *http.Request) { scrubber othewise bitd */ volinfo.Options[keyFeaturesScrub] = "true" - // Transaction which starts bitd and scrubber on all nodes. - txn := transaction.NewTxn(ctx) - defer txn.Done() - if err := txn.Ctx.Set("volinfo", volinfo); err != nil { logger.WithError(err).Error("failed to set volinfo in transaction context") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - //Lock on Volume Name - lock, unlock, err := transaction.CreateLockSteps(volName) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = volinfo.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -84,12 +83,10 @@ func bitrotEnableHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "vol-option.NotifyVolfileChange", Nodes: txn.Nodes, }, - { DoFunc: "bitrot-enable.Commit", Nodes: txn.Nodes, }, - unlock, } err = txn.Do() @@ -116,6 +113,17 @@ func bitrotDisableHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volName) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Validate volume existence volinfo, err := volume.GetVolume(volName) if err != nil { @@ -138,26 +146,14 @@ func bitrotDisableHandler(w http.ResponseWriter, r *http.Request) { // Disable scrub by updating volinfo Options volinfo.Options[keyFeaturesScrub] = "false" - // Transaction which stop bitd and scrubber on all nodes. - txn := transaction.NewTxn(ctx) - defer txn.Done() - if err := txn.Ctx.Set("volinfo", volinfo); err != nil { logger.WithError(err).Error("failed to set volinfo in transaction context") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - //Lock on Volume Name - lock, unlock, err := transaction.CreateLockSteps(volName) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = volinfo.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -171,7 +167,6 @@ func bitrotDisableHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "bitrot-disable.Commit", Nodes: txn.Nodes, }, - unlock, } err = txn.Do() @@ -196,6 +191,17 @@ func bitrotScrubOndemandHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volName) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Validate volume existence volinfo, err := volume.GetVolume(volName) if err != nil { @@ -219,25 +225,12 @@ func bitrotScrubOndemandHandler(w http.ResponseWriter, r *http.Request) { return } - // Transaction which starts scrubber on demand. - txn := transaction.NewTxn(ctx) - defer txn.Done() - - //Lock on Volume Name - lock, unlock, err := transaction.CreateLockSteps(volName) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = volinfo.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "bitrot-scrubondemand.Commit", Nodes: txn.Nodes, }, - unlock, } txn.Ctx.Set("volname", volName) @@ -261,6 +254,17 @@ func bitrotScrubStatusHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volName) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Validate volume existence volinfo, err := volume.GetVolume(volName) if err != nil { @@ -286,30 +290,16 @@ func bitrotScrubStatusHandler(w http.ResponseWriter, r *http.Request) { return } - // Transaction which gets scrubber status. - txn := transaction.NewTxn(ctx) - defer txn.Done() - - //Lock on Volume Name - lock, unlock, err := transaction.CreateLockSteps(volName) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, - err) - return - } - // Some nodes may not be up, which is okay. txn.DontCheckAlive = true txn.DisableRollback = true txn.Nodes = volinfo.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "bitrot-scrubstatus.Commit", Nodes: txn.Nodes, }, - unlock, } txn.Ctx.Set("volname", volName) diff --git a/plugins/georeplication/rest.go b/plugins/georeplication/rest.go index 0b3b6446b..190c6c332 100644 --- a/plugins/georeplication/rest.go +++ b/plugins/georeplication/rest.go @@ -110,6 +110,17 @@ func georepCreateHandler(w http.ResponseWriter, r *http.Request) { return } + txn, err := transaction.NewTxnWithLocks(ctx, req.MasterVol) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Check if Master volume exists and Matches with passed Volume ID vol, e := volume.GetVolume(req.MasterVol) if e != nil { @@ -158,17 +169,6 @@ func georepCreateHandler(w http.ResponseWriter, r *http.Request) { geoSession.Options["gluster-logdir"] = path.Join(config.GetString("logdir"), "glusterfs") } - // Transaction which updates the Geo-rep session - txn := transaction.NewTxn(ctx) - defer txn.Done() - - // Lock on Master Volume name - lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - // Required Volume Options vol.Options["marker.xtime"] = "on" vol.Options["marker.gsync-force-xtime"] = "on" @@ -179,7 +179,6 @@ func georepCreateHandler(w http.ResponseWriter, r *http.Request) { txn.Nodes = vol.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -192,7 +191,6 @@ func georepCreateHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "georeplication-create.Commit", Nodes: []uuid.UUID{gdctx.MyUUID}, }, - unlock, } if err = txn.Ctx.Set("geosession", geoSession); err != nil { @@ -275,6 +273,17 @@ func georepActionHandler(w http.ResponseWriter, r *http.Request, action actionTy return } + txn, err := transaction.NewTxnWithLocks(ctx, geoSession.MasterVol) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Fetch Volume details and check if Volume is in started state vol, e := volume.GetVolume(geoSession.MasterVol) if e != nil { @@ -291,15 +300,6 @@ func georepActionHandler(w http.ResponseWriter, r *http.Request, action actionTy return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - - lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - doFunc := "" stateToSet := "" var eventToSet georepEvent @@ -326,12 +326,10 @@ func georepActionHandler(w http.ResponseWriter, r *http.Request, action actionTy } txn.Steps = []*transaction.Step{ - lock, { DoFunc: doFunc, Nodes: vol.Nodes(), }, - unlock, } if err = txn.Ctx.Set("mastervolid", masterid.String()); err != nil { @@ -412,6 +410,17 @@ func georepDeleteHandler(w http.ResponseWriter, r *http.Request) { return } + txn, err := transaction.NewTxnWithLocks(ctx, geoSession.MasterVol) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Fetch Volume details and check if Volume exists _, e := volume.GetVolume(geoSession.MasterVol) if e != nil { @@ -423,23 +432,12 @@ func georepDeleteHandler(w http.ResponseWriter, r *http.Request) { return } - txn := transaction.NewTxn(ctx) - defer txn.Done() - - lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - // TODO: Add transaction step to clean xattrs specific to georep session txn.Steps = []*transaction.Step{ - lock, { DoFunc: "georeplication-delete.Commit", Nodes: []uuid.UUID{gdctx.MyUUID}, }, - unlock, } if err = txn.Ctx.Set("mastervolid", masterid.String()); err != nil { @@ -759,6 +757,17 @@ func georepConfigSetHandler(w http.ResponseWriter, r *http.Request) { } } + txn, err := transaction.NewTxnWithLocks(ctx, geoSession.MasterVol) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + vol, e := volume.GetVolume(geoSession.MasterVol) if e != nil { if err == errors.ErrVolNotFound { @@ -784,18 +793,8 @@ func georepConfigSetHandler(w http.ResponseWriter, r *http.Request) { geoSession.Options[k] = v } - txn := transaction.NewTxn(ctx) - defer txn.Done() - // TODO: change the lock key - lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = vol.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "georeplication-configset.Commit", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -804,7 +803,6 @@ func georepConfigSetHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "georeplication-configfilegen.Commit", Nodes: txn.Nodes, }, - unlock, } if err = txn.Ctx.Set("mastervolid", masterid.String()); err != nil { @@ -912,6 +910,17 @@ func georepConfigResetHandler(w http.ResponseWriter, r *http.Request) { restartRequired = false } + txn, err := transaction.NewTxnWithLocks(ctx, geoSession.MasterVol) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + vol, e := volume.GetVolume(geoSession.MasterVol) if e != nil { if err == errors.ErrVolNotFound { @@ -926,18 +935,8 @@ func georepConfigResetHandler(w http.ResponseWriter, r *http.Request) { delete(geoSession.Options, k) } - txn := transaction.NewTxn(ctx) - defer txn.Done() - // TODO: change the lock key - lock, unlock, err := transaction.CreateLockSteps(geoSession.MasterVol) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = vol.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "georeplication-configset.Commit", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -946,7 +945,6 @@ func georepConfigResetHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "georeplication-configfilegen.Commit", Nodes: txn.Nodes, }, - unlock, } if err = txn.Ctx.Set("mastervolid", masterid.String()); err != nil { @@ -1011,6 +1009,17 @@ func georepSSHKeyGenerateHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Check if Volume exists vol, e := volume.GetVolume(volname) if e != nil { @@ -1022,25 +1031,12 @@ func georepSSHKeyGenerateHandler(w http.ResponseWriter, r *http.Request) { return } - // Transaction which updates the Geo-rep session - txn := transaction.NewTxn(ctx) - defer txn.Done() - - // Lock on Master Volume name - lock, unlock, err := transaction.CreateLockSteps(volname) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = vol.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "georeplication-ssh-keygen.Commit", Nodes: txn.Nodes, }, - unlock, } if err = txn.Ctx.Set("volname", volname); err != nil { @@ -1100,6 +1096,17 @@ func georepSSHKeyPushHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Check if Volume exists vol, e := volume.GetVolume(volname) if e != nil { @@ -1118,25 +1125,12 @@ func georepSSHKeyPushHandler(w http.ResponseWriter, r *http.Request) { return } - // Transaction which updates the Geo-rep session - txn := transaction.NewTxn(ctx) - defer txn.Done() - - // Lock on Master Volume name - lock, unlock, err := transaction.CreateLockSteps(volname) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Nodes = vol.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "georeplication-ssh-keypush.Commit", Nodes: txn.Nodes, }, - unlock, } if err = txn.Ctx.Set("sshkeys", sshkeys); err != nil { diff --git a/plugins/glustershd/rest.go b/plugins/glustershd/rest.go index 5c05deec3..ed65ecab5 100644 --- a/plugins/glustershd/rest.go +++ b/plugins/glustershd/rest.go @@ -30,6 +30,17 @@ func glustershEnableHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + //validate volume name v, err := volume.GetVolume(volname) if err != nil { @@ -56,22 +67,10 @@ func glustershEnableHandler(w http.ResponseWriter, r *http.Request) { } - // Transaction which starts self heal daemon on all nodes with atleast one brick. - txn := transaction.NewTxn(ctx) - defer txn.Done() - - //Lock on Volume Name - lock, unlock, err := transaction.CreateLockSteps(volname) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - v.HealEnabled = true txn.Nodes = v.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -85,7 +84,6 @@ func glustershEnableHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "vol-option.NotifyVolfileChange", Nodes: txn.Nodes, }, - unlock, } if err := txn.Ctx.Set("volinfo", v); err != nil { @@ -117,6 +115,17 @@ func glustershDisableHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volname) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + //validate volume name v, err := volume.GetVolume(volname) if err != nil { @@ -137,23 +146,10 @@ func glustershDisableHandler(w http.ResponseWriter, r *http.Request) { return } - // Transaction which checks if all replicate volumes are stopped before - // stopping the self-heal daemon. - txn := transaction.NewTxn(ctx) - defer txn.Done() - - // Lock on volume name. - lock, unlock, err := transaction.CreateLockSteps(volname) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - v.HealEnabled = false txn.Nodes = v.Nodes() txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -168,7 +164,6 @@ func glustershDisableHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "vol-option.NotifyVolfileChange", Nodes: txn.Nodes, }, - unlock, } if err := txn.Ctx.Set("volinfo", v); err != nil { diff --git a/plugins/quota/rest.go b/plugins/quota/rest.go index f34ae3d45..1e0fb1a18 100644 --- a/plugins/quota/rest.go +++ b/plugins/quota/rest.go @@ -47,6 +47,17 @@ func quotaEnableHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := gdctx.GetReqLogger(ctx) + txn, err := transaction.NewTxnWithLocks(ctx, volName) + if err != nil { + if err == transaction.ErrLockTimeout { + restutils.SendHTTPError(ctx, w, http.StatusConflict, err) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) + } + return + } + defer txn.Done() + // Validate volume existence vol, err := volume.GetVolume(volName) if err != nil { @@ -73,23 +84,13 @@ func quotaEnableHandler(w http.ResponseWriter, r *http.Request) { // Enable quota vol.Options[quotaEnabledKey] = "on" - txn := transaction.NewTxn(ctx) - defer txn.Done() - if err := txn.Ctx.Set("volinfo", vol); err != nil { logger.WithError(err).Error("failed to set volinfo in transaction context") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - lock, unlock, err := transaction.CreateLockSteps(volName) - if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - txn.Steps = []*transaction.Step{ - lock, { DoFunc: "vol-option.UpdateVolinfo", Nodes: []uuid.UUID{gdctx.MyUUID}, @@ -98,7 +99,6 @@ func quotaEnableHandler(w http.ResponseWriter, r *http.Request) { DoFunc: "quota-enable.DaemonStart", Nodes: vol.Nodes(), }, - unlock, } err = txn.Do() From 21d204eecacf8a5f41b9ba7b5d5b470ad36399f3 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 28 May 2018 21:27:56 +0530 Subject: [PATCH 49/54] nightly-rpms: Use /usr/bin/mock instead of mock Using `mock` on CentOS/Fedora as root resolves it to /sbin/mock, which only works if a mock user has been setup. Forcing the use of /usr/bin/mock makes this work always. --- extras/nightly-rpms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/nightly-rpms.sh b/extras/nightly-rpms.sh index e011cdad6..6ed824db0 100755 --- a/extras/nightly-rpms.sh +++ b/extras/nightly-rpms.sh @@ -73,7 +73,7 @@ SRPM=$(rpmbuild --define "_topdir $PWD/rpmbuild" -bs rpmbuild/SPECS/$SPEC | cut # Build RPM from SRPM using mock mkdir -p "$RESULTDIR" -mock -r epel-7-x86_64 --resultdir="$RESULTDIR" --rebuild "$SRPM" +/usr/bin/mock -r epel-7-x86_64 --resultdir="$RESULTDIR" --rebuild "$SRPM" popd #BUILDDIR From d7bfb63b0e12794cd0c4beff23e7aec639b89e74 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 28 May 2018 21:21:09 +0530 Subject: [PATCH 50/54] Add back the old transaction/lock functions ... so that PRs already present can be merged. --- glusterd2/transaction/lock.go | 126 +++++++++++++++++++++++++++ glusterd2/transaction/transaction.go | 12 ++- 2 files changed, 135 insertions(+), 3 deletions(-) diff --git a/glusterd2/transaction/lock.go b/glusterd2/transaction/lock.go index 1ed436a66..4fe6b0c65 100644 --- a/glusterd2/transaction/lock.go +++ b/glusterd2/transaction/lock.go @@ -5,9 +5,12 @@ import ( "errors" "time" + "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/store" "github.com/coreos/etcd/clientv3/concurrency" + "github.com/pborman/uuid" + log "github.com/sirupsen/logrus" ) const ( @@ -23,6 +26,129 @@ var ( ErrLockExists = errors.New("existing lock found for given lock ID") ) +// createLockStepFunc returns the registry IDs of StepFuncs which lock/unlock the given key. +// If existing StepFuncs are not found, new funcs are created and registered. +func createLockStepFunc(key string) (string, string, error) { + lockFuncID := key + ".Lock" + unlockFuncID := key + ".Unlock" + + _, lockFuncFound := getStepFunc(lockFuncID) + _, unlockFuncFound := getStepFunc(unlockFuncID) + + if lockFuncFound && unlockFuncFound { + return lockFuncID, unlockFuncID, nil + } + + key = lockPrefix + key + locker := concurrency.NewMutex(store.Store.Session, key) + + lockFunc := func(c TxnCtx) error { + + ctx, cancel := context.WithTimeout(context.Background(), lockObtainTimeout) + defer cancel() + + c.Logger().WithField("key", key).Debug("attempting to lock") + err := locker.Lock(ctx) + switch err { + case nil: + c.Logger().WithField("key", key).Debug("lock obtained") + case context.DeadlineExceeded: + // Propagate this all the way back to the client as a HTTP 409 response + c.Logger().WithField("key", key).Debug("timeout: failed to obtain lock") + err = ErrLockTimeout + } + + return err + } + RegisterStepFunc(lockFunc, lockFuncID) + + unlockFunc := func(c TxnCtx) error { + + c.Logger().WithField("key", key).Debug("attempting to unlock") + err := locker.Unlock(context.Background()) + if err == nil { + c.Logger().WithField("key", key).Debug("lock unlocked") + } + + return err + } + RegisterStepFunc(unlockFunc, unlockFuncID) + + return lockFuncID, unlockFuncID, nil +} + +// CreateLockSteps returns a lock and an unlock Step which lock/unlock the given key +// TODO: Remove this function +func CreateLockSteps(key string) (*Step, *Step, error) { + lockFunc, unlockFunc, err := createLockStepFunc(key) + if err != nil { + return nil, nil, err + } + + lockStep := &Step{lockFunc, unlockFunc, []uuid.UUID{gdctx.MyUUID}, false} + unlockStep := &Step{unlockFunc, "", []uuid.UUID{gdctx.MyUUID}, false} + + return lockStep, unlockStep, nil +} + +// LockUnlockFunc is signature of functions used for distributed locking +// and unlocking. +type LockUnlockFunc func(ctx context.Context) error + +// CreateLockFuncs creates and returns functions for distributed lock and +// unlock. This is similar to CreateLockSteps() but returns normal functions. +// TODO: Remove this function +func CreateLockFuncs(key string) (LockUnlockFunc, LockUnlockFunc) { + + key = lockPrefix + key + locker := concurrency.NewMutex(store.Store.Session, key) + + // TODO: There is an opportunity for refactor here to re-use code + // between CreateLockFunc and CreateLockSteps. This variant doesn't + // have registry either. + + lockFunc := func(ctx context.Context) error { + logger := gdctx.GetReqLogger(ctx) + if logger == nil { + logger = log.StandardLogger() + } + + ctx, cancel := context.WithTimeout(ctx, lockObtainTimeout) + defer cancel() + + logger.WithField("key", key).Debug("attempting to lock") + err := locker.Lock(ctx) + switch err { + case nil: + logger.WithField("key", key).Debug("lock obtained") + case context.DeadlineExceeded: + // Propagate this all the way back to the client as a HTTP 409 response + logger.WithField("key", key).Debug("timeout: failed to obtain lock") + err = ErrLockTimeout + } + + return err + } + + unlockFunc := func(ctx context.Context) error { + logger := gdctx.GetReqLogger(ctx) + if logger == nil { + logger = log.StandardLogger() + } + + logger.WithField("key", key).Debug("attempting to unlock") + if err := locker.Unlock(context.Background()); err != nil { + logger.WithField("key", key).WithError(err).Error("unlock failed") + return err + } + + logger.WithField("key", key).Debug("lock unlocked") + return nil + } + + return lockFunc, unlockFunc +} + func (t *Txn) lock(lockID string) error { // Ensure that no prior lock exists for the given lockID in this transaction if _, ok := t.locks[lockID]; ok { diff --git a/glusterd2/transaction/transaction.go b/glusterd2/transaction/transaction.go index 902561393..96b1e58e2 100644 --- a/glusterd2/transaction/transaction.go +++ b/glusterd2/transaction/transaction.go @@ -54,7 +54,6 @@ func NewTxn(ctx context.Context) *Txn { }).WithPrefix(prefix) t.Ctx.Logger().Debug("new transaction created") - expTxn.Add("initiated_txn_in_progress", 1) return t } @@ -72,6 +71,13 @@ func NewTxnWithLocks(ctx context.Context, lockIDs ...string) (*Txn, error) { return t, nil } +// Cleanup cleans the leftovers after a transaction ends +// TODO: Remove this function +func (t *Txn) Cleanup() { + store.Store.Delete(context.TODO(), t.Ctx.Prefix(), clientv3.WithPrefix()) + expTxn.Add("initiated_txn_in_progress", -1) +} + // Done releases any obtained locks and cleans up the transaction namespace // Done must be called after a transaction ends func (t *Txn) Done() { @@ -79,8 +85,7 @@ func (t *Txn) Done() { for _, locker := range t.locks { locker.Unlock(context.Background()) } - store.Store.Delete(context.TODO(), t.Ctx.Prefix(), clientv3.WithPrefix()) - expTxn.Add("initiated_txn_in_progress", -1) + t.Cleanup() } func (t *Txn) checkAlive() error { @@ -111,6 +116,7 @@ func (t *Txn) Do() error { } t.Ctx.Logger().Debug("Starting transaction") + expTxn.Add("initiated_txn_in_progress", 1) for i, s := range t.Steps { if s.Skip { From 203f5ea5e9294d6236fa4e10b810a9b7b56a57bd Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 15 May 2018 10:52:07 +0530 Subject: [PATCH 51/54] added functionality to expose flags in volume create and volume expand added additional flag to create brick dir if its not exists exposed flags for volume create and expand in glustercli added e2e test cases for flags in volume create and volume expand Signed-off-by: Madhu Rajanna --- e2e/volume_ops_test.go | 77 +++++++++++++++++-- glustercli/cmd/volume-create.go | 15 +++- glustercli/cmd/volume.go | 14 ++++ glusterd2/brick/types.go | 11 +++ glusterd2/brick/validation.go | 41 +++++++++- glusterd2/commands/volumes/common.go | 17 ++++ .../commands/volumes/volume-create-txn.go | 10 +-- glusterd2/commands/volumes/volume-create.go | 2 +- .../commands/volumes/volume-expand-txn.go | 10 +-- glusterd2/commands/volumes/volume-expand.go | 8 +- pkg/api/volume_req.go | 20 ++++- pkg/errors/error.go | 1 + 12 files changed, 193 insertions(+), 33 deletions(-) diff --git a/e2e/volume_ops_test.go b/e2e/volume_ops_test.go index 469a0aa27..8fce68e06 100644 --- a/e2e/volume_ops_test.go +++ b/e2e/volume_ops_test.go @@ -48,7 +48,6 @@ func TestVolume(t *testing.T) { // Create the volume t.Run("Create", testVolumeCreate) - // Expand the volume t.Run("Expand", testVolumeExpand) @@ -112,18 +111,84 @@ func testVolumeCreate(t *testing.T) { createReq.Name = "##@@#@!#@!!@#" _, err = client.VolumeCreate(createReq) r.NotNil(err) + } +func testVolumeCreateWithFlags(t *testing.T) { + r := require.New(t) + + var brickPaths []string + + for i := 1; i <= 4; i++ { + brickPaths = append(brickPaths, fmt.Sprintf(baseWorkdir+t.Name()+"/b/%d", i)) + } + + flags := make(map[string]bool) + //set flags to allow rootdir + flags["allow-root-dir"] = true + //set flags create brick dir + flags["create-brick-dir"] = true + + createReqBrick := api.VolCreateReq{ + Name: t.Name(), + Subvols: []api.SubvolReq{ + { + ReplicaCount: 2, + Type: "replicate", + Bricks: []api.BrickReq{ + {PeerID: gds[0].PeerID(), Path: brickPaths[0]}, + {PeerID: gds[1].PeerID(), Path: brickPaths[1]}, + }, + }, + { + Type: "replicate", + ReplicaCount: 2, + Bricks: []api.BrickReq{ + {PeerID: gds[0].PeerID(), Path: brickPaths[2]}, + {PeerID: gds[1].PeerID(), Path: brickPaths[3]}, + }, + }, + }, + Flags: flags, + } + + _, err := client.VolumeCreate(createReqBrick) + r.Nil(err) + + //delete volume + r.Nil(client.VolumeDelete(t.Name())) + + createReqBrick.Name = t.Name() + //set reuse-brick flag + flags["reuse-bricks"] = true + createReqBrick.Flags = flags + + _, err = client.VolumeCreate(createReqBrick) + r.Nil(err) + + r.Nil(client.VolumeDelete(t.Name())) + + //recreate deleted volume + _, err = client.VolumeCreate(createReqBrick) + r.Nil(err) + + //delete volume + r.Nil(client.VolumeDelete(t.Name())) + +} func testVolumeExpand(t *testing.T) { r := require.New(t) var brickPaths []string for i := 1; i <= 4; i++ { - brickPath, err := ioutil.TempDir(tmpDir, "brick") - r.Nil(err) - brickPaths = append(brickPaths, brickPath) + brickPaths = append(brickPaths, fmt.Sprintf(fmt.Sprintf(baseWorkdir+t.Name()+"/b/%d/", i))) } + flags := make(map[string]bool) + //set flags to allow rootdir and create brick dir + flags["create-brick-dir"] = true + flags["allow-root-dir"] = true + expandReq := api.VolExpandReq{ Bricks: []api.BrickReq{ {PeerID: gds[0].PeerID(), Path: brickPaths[0]}, @@ -131,8 +196,10 @@ func testVolumeExpand(t *testing.T) { {PeerID: gds[0].PeerID(), Path: brickPaths[2]}, {PeerID: gds[1].PeerID(), Path: brickPaths[3]}, }, - Force: true, + Flags: flags, } + + //expand with new brick dir which is not created _, err := client.VolumeExpand(volname, expandReq) r.Nil(err) } diff --git a/glustercli/cmd/volume-create.go b/glustercli/cmd/volume-create.go index b48755fa3..c40d54c38 100644 --- a/glustercli/cmd/volume-create.go +++ b/glustercli/cmd/volume-create.go @@ -48,9 +48,11 @@ func init() { volumeCreateCmd.Flags().StringSliceVar(&flagCreateVolumeOptions, "options", []string{}, "Volume options in the format option:value,option:value") volumeCreateCmd.Flags().BoolVar(&flagCreateAdvOpts, "advanced", false, "Allow advanced options") - volumeCreateCmd.Flags().BoolVar(&flagCreateExpOpts, "experimental", false, "Allow experimental options") - volumeCreateCmd.Flags().BoolVar(&flagCreateDepOpts, "deprecated", false, "Allow deprecated options") - volumeCmd.AddCommand(volumeCreateCmd) + volumeCreateCmd.Flags().BoolVar(&flagReuseBricks, "reuse-bricks", false, "Reuse bricks") + volumeCreateCmd.Flags().BoolVar(&flagAllowRootDir, "allow-root-dir", false, "Allow root directory") + volumeCreateCmd.Flags().BoolVar(&flagAllowMountAsBrick, "allow-mount-as-brick", false, "Allow mount as bricks") + volumeCreateCmd.Flags().BoolVar(&flagCreateBrickDir, "create-brick-dir", false, "Create brick directory") + } func volumeCreateCmdRun(cmd *cobra.Command, args []string) { @@ -133,6 +135,12 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { ) } } + //set flags + flags := make(map[string]bool) + flags["reuse-bricks"] = flagReuseBricks + flags["allow-root-dir"] = flagAllowRootDir + flags["allow-mount-as-brick"] = flagAllowMountAsBrick + flags["create-brick-dir"] = flagCreateBrickDir options := make(map[string]string) //set options @@ -161,6 +169,7 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { Advanced: flagCreateAdvOpts, Experimental: flagCreateExpOpts, Deprecated: flagCreateDepOpts, + Flags: flags, } // handle thin-arbiter diff --git a/glustercli/cmd/volume.go b/glustercli/cmd/volume.go index 6a1c43de1..f60528dc0 100644 --- a/glustercli/cmd/volume.go +++ b/glustercli/cmd/volume.go @@ -49,6 +49,8 @@ var ( flagCmdMetadataKey string flagCmdMetadataValue string flagCmdDeleteMetadata bool + //volume expand flags + flagReuseBricks, flagAllowRootDir, flagAllowMountAsBrick, flagCreateBrickDir bool ) func init() { @@ -79,6 +81,10 @@ func init() { // Volume Expand volumeExpandCmd.Flags().IntVarP(&flagExpandCmdReplicaCount, "replica", "", 0, "Replica Count") volumeExpandCmd.Flags().BoolVarP(&flagExpandCmdForce, "force", "f", false, "Force") + volumeExpandCmd.Flags().BoolVar(&flagReuseBricks, "reuse-bricks", false, "Reuse Bricks") + volumeExpandCmd.Flags().BoolVar(&flagAllowRootDir, "allow-root-dir", false, "Allow Root Directory") + volumeExpandCmd.Flags().BoolVar(&flagAllowMountAsBrick, "allow-mount-as-brick", false, "Allow Mount as Bricks") + volumeExpandCmd.Flags().BoolVar(&flagCreateBrickDir, "create-brick-dir", false, "Create brick directory") volumeCmd.AddCommand(volumeExpandCmd) // Volume Edit @@ -440,10 +446,18 @@ var volumeExpandCmd = &cobra.Command{ } failure("Error getting brick UUIDs", err, 1) } + //set flags + flags := make(map[string]bool) + flags["reuse-bricks"] = flagReuseBricks + flags["allow-root-dir"] = flagAllowRootDir + flags["allow-mount-as-brick"] = flagAllowMountAsBrick + flags["create-brick-dir"] = flagCreateBrickDir + vol, err := client.VolumeExpand(volname, api.VolExpandReq{ ReplicaCount: flagExpandCmdReplicaCount, Bricks: bricks, // string of format : Force: flagExpandCmdForce, + Flags: flags, }) if err != nil { if verbose { diff --git a/glusterd2/brick/types.go b/glusterd2/brick/types.go index 025f186f2..52e464685 100644 --- a/glusterd2/brick/types.go +++ b/glusterd2/brick/types.go @@ -2,6 +2,7 @@ package brick import ( "fmt" + "os" "github.com/pborman/uuid" "golang.org/x/sys/unix" @@ -80,6 +81,16 @@ func (b *Brickinfo) Validate(check InitChecks) error { return err } + if _, err = os.Stat(b.Path); os.IsNotExist(err) { + if check.CreateBrickDir { + if err = os.MkdirAll(b.Path, 0775); err != nil { + return err + } + } else { + return err + } + } + if err = unix.Lstat(b.Path, &brickStat); err != nil { return err } diff --git a/glusterd2/brick/validation.go b/glusterd2/brick/validation.go index 78c372606..4252257f4 100644 --- a/glusterd2/brick/validation.go +++ b/glusterd2/brick/validation.go @@ -20,11 +20,46 @@ const ( // InitChecks is a set of checks to be run on a brick type InitChecks struct { - IsInUse bool - IsMount bool - IsOnRoot bool + IsInUse bool + IsMount bool + IsOnRoot bool + CreateBrickDir bool } +// PrepareChecks initializes InitChecks based on req +func PrepareChecks(force bool, req map[string]bool) *InitChecks { + c := &InitChecks{} + + if force { + // skip all checks if force is set to true + + c.CreateBrickDir = true + return c + } + + // do all the checks except the ones explicitly excluded + c.IsInUse = true + c.IsOnRoot = true + c.IsMount = true + c.CreateBrickDir = false + + if value, ok := req["reuse-bricks"]; ok && value { + c.IsInUse = false + } + + if value, ok := req["allow-root-dir"]; ok && value { + c.IsOnRoot = false + } + + if value, ok := req["allow-mount-as-brick"]; ok && value { + c.IsMount = false + } + if value, ok := req["create-brick-dir"]; ok && value { + c.CreateBrickDir = true + } + + return c +} func validatePathLength(path string) error { if len(filepath.Clean(path)) >= syscall.PathMax { diff --git a/glusterd2/commands/volumes/common.go b/glusterd2/commands/volumes/common.go index 92802f3c7..1a2c3af3c 100644 --- a/glusterd2/commands/volumes/common.go +++ b/glusterd2/commands/volumes/common.go @@ -18,6 +18,7 @@ import ( "github.com/gluster/glusterd2/glusterd2/xlator" "github.com/gluster/glusterd2/glusterd2/xlator/options" "github.com/gluster/glusterd2/pkg/api" + gderrors "github.com/gluster/glusterd2/pkg/errors" "github.com/pborman/uuid" log "github.com/sirupsen/logrus" @@ -264,3 +265,19 @@ func LoadDefaultGroupOptions() error { } return nil } + +//validateVolumeFlags checks for Flags in volume create and expand +func validateVolumeFlags(flag map[string]bool) error { + if len(flag) > 4 { + return gderrors.ErrInvalidVolFlags + } + for key := range flag { + switch key { + case "reuse-bricks", "allow-root-dir", "allow-mount-as-brick", "create-brick-dir": + continue + default: + return fmt.Errorf("volume flag not supported %s", key) + } + } + return nil +} diff --git a/glusterd2/commands/volumes/volume-create-txn.go b/glusterd2/commands/volumes/volume-create-txn.go index 351441f0b..015c68e4b 100644 --- a/glusterd2/commands/volumes/volume-create-txn.go +++ b/glusterd2/commands/volumes/volume-create-txn.go @@ -194,15 +194,9 @@ func createVolinfo(c transaction.TxnCtx) error { return err } - // TODO: Expose this granularity in the volume create API. - var checks brick.InitChecks - if !req.Force { - checks.IsInUse = true - checks.IsMount = true - checks.IsOnRoot = true - } + checks := brick.PrepareChecks(req.Force, req.Flags) - err = c.Set("brick-checks", &checks) + err = c.Set("brick-checks", checks) return err } diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index d9955737d..415f0f359 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -43,7 +43,7 @@ func validateVolCreateReq(req *api.VolCreateReq) error { } } - return nil + return validateVolumeFlags(req.Flags) } func checkDupBrickEntryVolCreate(req api.VolCreateReq) error { diff --git a/glusterd2/commands/volumes/volume-expand-txn.go b/glusterd2/commands/volumes/volume-expand-txn.go index 07b4078e9..ff8620c44 100644 --- a/glusterd2/commands/volumes/volume-expand-txn.go +++ b/glusterd2/commands/volumes/volume-expand-txn.go @@ -59,13 +59,9 @@ func expandValidatePrepare(c transaction.TxnCtx) error { return err } - var checks brick.InitChecks - if !req.Force { - checks.IsInUse = true - checks.IsMount = true - checks.IsOnRoot = true - } - if err := c.Set("brick-checks", &checks); err != nil { + checks := brick.PrepareChecks(req.Force, req.Flags) + + if err := c.Set("brick-checks", checks); err != nil { return err } diff --git a/glusterd2/commands/volumes/volume-expand.go b/glusterd2/commands/volumes/volume-expand.go index e8a190253..891947520 100644 --- a/glusterd2/commands/volumes/volume-expand.go +++ b/glusterd2/commands/volumes/volume-expand.go @@ -35,7 +35,8 @@ func registerVolExpandStepFuncs() { transaction.RegisterStepFunc(sf.sf, sf.name) } } -func checkDupBrickEntryVolExpand(req api.VolExpandReq) error { + +func validateVolumeExpandReq(req api.VolExpandReq) error { dupEntry := map[string]bool{} for _, brick := range req.Bricks { @@ -46,7 +47,8 @@ func checkDupBrickEntryVolExpand(req api.VolExpandReq) error { } - return nil + return validateVolumeFlags(req.Flags) + } func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { @@ -61,7 +63,7 @@ func volumeExpandHandler(w http.ResponseWriter, r *http.Request) { return } - if err := checkDupBrickEntryVolExpand(req); err != nil { + if err := validateVolumeExpandReq(req); err != nil { restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) return } diff --git a/pkg/api/volume_req.go b/pkg/api/volume_req.go index d1fa8e946..0cf4c75e6 100644 --- a/pkg/api/volume_req.go +++ b/pkg/api/volume_req.go @@ -20,6 +20,12 @@ type SubvolReq struct { } // VolCreateReq represents a Volume Create Request +/*supported Flags +"reuse-bricks" : for reusing of bricks +"allow-root-dir" : allow root directory to create brick +"allow-mount-as-brick" : reuse if its already mountpoint +"create-brick-dir" : if brick dir is not present, create it +*/ type VolCreateReq struct { Name string `json:"name"` Transport string `json:"transport,omitempty"` @@ -30,6 +36,7 @@ type VolCreateReq struct { Experimental bool `json:"experimental,omitempty"` Deprecated bool `json:"deprecated,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` + Flags map[string]bool `json:"flags,omitempty"` } // VolOptionReq represents an incoming request to set volume options @@ -47,10 +54,17 @@ type VolOptionResetReq struct { } // VolExpandReq represents a request to expand the volume by adding more bricks +/*supported Flags +"reuse-bricks" : for reusing of bricks +"allow-root-dir" : allow root directory to create brick +"allow-mount-as-brick" : reuse if its already mountpoint +"create-brick-dir" : if brick dir is not present, create it +*/ type VolExpandReq struct { - ReplicaCount int `json:"replica,omitempty"` - Bricks []BrickReq `json:"bricks"` - Force bool `json:"force,omitempty"` + ReplicaCount int `json:"replica,omitempty"` + Bricks []BrickReq `json:"bricks"` + Force bool `json:"force,omitempty"` + Flags map[string]bool `json:"flags,omitempty"` } // VolumeOption represents an option that is part of a profile diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 7eb700596..bb9c2fbf3 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -42,4 +42,5 @@ var ( ErrDuplicateBrickPath = errors.New("Duplicate brick entry") ErrRestrictedKeyFound = errors.New("Key names starting with '_' are restricted in metadata field") ErrVolFileNotFound = errors.New("volume file not found") + ErrInvalidVolFlags = errors.New("invalid volume flags") ) From f76d86cc2bf4e5fc5d31937315111eb18bf8b121 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 29 May 2018 12:09:01 +0530 Subject: [PATCH 52/54] add volume create to glustercli Signed-off-by: Madhu Rajanna --- glustercli/cmd/volume-create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glustercli/cmd/volume-create.go b/glustercli/cmd/volume-create.go index c40d54c38..9cd58fd38 100644 --- a/glustercli/cmd/volume-create.go +++ b/glustercli/cmd/volume-create.go @@ -52,7 +52,7 @@ func init() { volumeCreateCmd.Flags().BoolVar(&flagAllowRootDir, "allow-root-dir", false, "Allow root directory") volumeCreateCmd.Flags().BoolVar(&flagAllowMountAsBrick, "allow-mount-as-brick", false, "Allow mount as bricks") volumeCreateCmd.Flags().BoolVar(&flagCreateBrickDir, "create-brick-dir", false, "Create brick directory") - + volumeCmd.AddCommand(volumeCreateCmd) } func volumeCreateCmdRun(cmd *cobra.Command, args []string) { From 501556e005aed2f6ef063958a5ed02a5b012dfe3 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 29 May 2018 13:56:51 +0530 Subject: [PATCH 53/54] Display peer addresses in new line When there are multiple peer addresses, display each one in a separate line. Signed-off-by: Madhu Rajanna --- glustercli/cmd/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glustercli/cmd/peer.go b/glustercli/cmd/peer.go index ce09d006d..04508f3be 100644 --- a/glustercli/cmd/peer.go +++ b/glustercli/cmd/peer.go @@ -70,7 +70,7 @@ var peerAddCmd = &cobra.Command{ fmt.Println("Peer add successful") table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"ID", "Name", "Peer Addresses"}) - table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, ",")}) + table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, "\n")}) table.Render() }, } @@ -123,7 +123,7 @@ func peerStatusHandler(cmd *cobra.Command) { table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"ID", "Name", "Peer Addresses", "Online"}) for _, peer := range peers { - table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, ","), formatBoolYesNo(peer.Online)}) + table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, "\n"), formatBoolYesNo(peer.Online)}) } table.Render() } From 6bc5318b4b259b7da54d5bc995d67e3f5cb227e7 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 29 May 2018 16:08:07 +0530 Subject: [PATCH 54/54] added missed experimental and deprecated flags Signed-off-by: Madhu Rajanna --- glustercli/cmd/volume-create.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glustercli/cmd/volume-create.go b/glustercli/cmd/volume-create.go index 9cd58fd38..9202f09b6 100644 --- a/glustercli/cmd/volume-create.go +++ b/glustercli/cmd/volume-create.go @@ -48,6 +48,8 @@ func init() { volumeCreateCmd.Flags().StringSliceVar(&flagCreateVolumeOptions, "options", []string{}, "Volume options in the format option:value,option:value") volumeCreateCmd.Flags().BoolVar(&flagCreateAdvOpts, "advanced", false, "Allow advanced options") + volumeCreateCmd.Flags().BoolVar(&flagCreateExpOpts, "experimental", false, "Allow experimental options") + volumeCreateCmd.Flags().BoolVar(&flagCreateDepOpts, "deprecated", false, "Allow deprecated options") volumeCreateCmd.Flags().BoolVar(&flagReuseBricks, "reuse-bricks", false, "Reuse bricks") volumeCreateCmd.Flags().BoolVar(&flagAllowRootDir, "allow-root-dir", false, "Allow root directory") volumeCreateCmd.Flags().BoolVar(&flagAllowMountAsBrick, "allow-mount-as-brick", false, "Allow mount as bricks")