diff --git a/doc/endpoints.md b/doc/endpoints.md index 81555c53c..d4427f1b7 100644 --- a/doc/endpoints.md +++ b/doc/endpoints.md @@ -31,6 +31,7 @@ VolumeStop | POST | /volumes/{volname}/stop | [](https://godoc.org/github.com/gl Statedump | POST | /volumes/{volname}/statedump | [VolStatedumpReq](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolStatedumpReq) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) VolfilesGenerate | POST | /volfiles | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) VolfilesGet | GET | /volfiles | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) +VolfilesGet | GET | /volfiles/{volfileid:.*} | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) EditVolume | POST | /volumes/{volname}/edit | [VolEditReq](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolEditReq) | [VolumeEditResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#VolumeEditResp) GetPeer | GET | /peers/{peerid} | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [PeerGetResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#PeerGetResp) GetPeers | GET | /peers | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [PeerListResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#PeerListResp) diff --git a/doc/quick-start-user-guide.md b/doc/quick-start-user-guide.md index b5ae506d6..9d0f4e928 100644 --- a/doc/quick-start-user-guide.md +++ b/doc/quick-start-user-guide.md @@ -83,9 +83,9 @@ INFO[2017-08-28T16:03:58+05:30] started GlusterD SunRPC server ip Now you have two nodes running glusterd2. -> NOTE: Ensure that firewalld is configured (or stopped) to let traffic on ports ` before attaching a peer. +> NOTE: Ensure that firewalld is configured (or stopped) to let traffic on ports ` before adding a peer. -## Attach peer +## Add peer Glusterd2 natively provides only ReST API for clients to perform management operations. A CLI is provided which interacts with glusterd2 using the [ReST APIs](https://github.com/gluster/glusterd2/wiki/ReST-API). @@ -99,7 +99,7 @@ $ cat addpeer.json "addresses": ["192.168.56.102"] } ``` -`addresses` takes a list of address by which the new host can be added. It can be FQDNs, short-names or IP addresses. Note that if you want to attach multiple peers use below API to attach each peer one at a time. +`addresses` takes a list of address by which the new host can be added. It can be FQDNs, short-names or IP addresses. Note that if you want to add multiple peers use below API to add each peer one at a time. Send a HTTP request to `node1` to add `node2` as peer: @@ -109,7 +109,7 @@ $ curl -X POST http://192.168.56.101:24007/v1/peers --data @addpeer.json -H 'Con or using glustercli: - $ glustercli peer probe 192.168.56.102 + $ glustercli peer add 192.168.56.102 You will get the Peer ID of the newly added peer as response. @@ -123,7 +123,7 @@ $ curl -X GET http://192.168.56.101:24007/v1/peers or by using the glustercli: - $ glustercli pool list + $ glustercli peer list Note the UUIDs in the response. We will use the same in volume create request below. @@ -199,4 +199,4 @@ Verify that `glusterfsd` process is running on both nodes. * Issues with 2 node clusters * Restarting glusterd2 does not restore the cluster - * Peer detach doesn't work + * Peer remove doesn't work diff --git a/e2e/peer_ops_test.go b/e2e/peer_ops_test.go index f614e4830..14d35897e 100644 --- a/e2e/peer_ops_test.go +++ b/e2e/peer_ops_test.go @@ -27,13 +27,13 @@ func TestAddRemovePeer(t *testing.T) { client := initRestclient(g1.ClientAddress) - _, err2 := client.PeerProbe(g2.PeerAddress) + _, err2 := client.PeerAdd(g2.PeerAddress) r.Nil(err2) time.Sleep(6 * time.Second) // add peer: ask g1 to add g3 as peer - _, err3 := client.PeerProbe(g3.PeerAddress) + _, err3 := client.PeerAdd(g3.PeerAddress) r.Nil(err3) time.Sleep(6 * time.Second) @@ -44,6 +44,6 @@ func TestAddRemovePeer(t *testing.T) { r.Len(peers, 3) // remove peer: ask g1 to remove g2 as peer - err5 := client.PeerDetach(g2.PeerID()) + err5 := client.PeerRemove(g2.PeerID()) r.Nil(err5) } diff --git a/e2e/volume_ops_test.go b/e2e/volume_ops_test.go index fad684bb2..e81201c2a 100644 --- a/e2e/volume_ops_test.go +++ b/e2e/volume_ops_test.go @@ -100,6 +100,9 @@ func testVolumeCreate(t *testing.T) { }, }, }, + Metadata: map[string]string{ + "owner": "gd2test", + }, Force: true, } _, err := client.VolumeCreate(createReq) @@ -148,6 +151,40 @@ func testVolumeStop(t *testing.T) { func testVolumeList(t *testing.T) { r := require.New(t) + var matchingQueries []map[string]string + var nonMatchingQueries []map[string]string + + matchingQueries = append(matchingQueries, map[string]string{ + "key": "owner", + "value": "gd2test", + }) + matchingQueries = append(matchingQueries, map[string]string{ + "key": "owner", + }) + matchingQueries = append(matchingQueries, map[string]string{ + "value": "gd2test", + }) + for _, filter := range matchingQueries { + volumes, err := client.Volumes("", filter) + r.Nil(err) + r.Len(volumes, 1) + } + + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "key": "owner", + "value": "gd2-test", + }) + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "key": "owners", + }) + nonMatchingQueries = append(nonMatchingQueries, map[string]string{ + "value": "gd2tests", + }) + for _, filter := range nonMatchingQueries { + volumes, err := client.Volumes("", filter) + r.Nil(err) + r.Len(volumes, 0) + } volumes, err := client.Volumes("") r.Nil(err) diff --git a/glustercli/cmd/bitrot.go b/glustercli/cmd/bitrot.go index 898d43911..5dbf1ac05 100644 --- a/glustercli/cmd/bitrot.go +++ b/glustercli/cmd/bitrot.go @@ -100,6 +100,8 @@ var bitrotScrubThrottleCmd = &cobra.Command{ option = append(option, "bit-rot.scrub-throttle") option = append(option, args[1]) + // Set Option set flag to advanced + flagSetAdv = true err := volumeOptionJSONHandler(cmd, volname, option) if err != nil { if verbose { @@ -125,6 +127,8 @@ var bitrotScrubFrequencyCmd = &cobra.Command{ option = append(option, "bit-rot.scrub-freq") option = append(option, args[1]) + // Set Option set flag to advanced + flagSetAdv = true err := volumeOptionJSONHandler(cmd, volname, option) if err != nil { if verbose { @@ -152,6 +156,9 @@ var bitrotScrubCmd = &cobra.Command{ case scrubPause, scrubResume: option = append(option, "bit-rot.scrub-state") option = append(option, args[1]) + + // Set Option set flag to advanced + flagSetAdv = true err := volumeOptionJSONHandler(cmd, volname, option) if err != nil { if verbose { diff --git a/glustercli/cmd/peer.go b/glustercli/cmd/peer.go index e042c1e4e..2d5d64c00 100644 --- a/glustercli/cmd/peer.go +++ b/glustercli/cmd/peer.go @@ -14,30 +14,29 @@ import ( const ( helpPeerCmd = "Gluster Peer Management" - helpPeerProbeCmd = "probe peer specified by " - helpPeerDetachCmd = "detach peer specified by " + helpPeerAddCmd = "add peer specified by " + helpPeerRemoveCmd = "remove peer specified by " helpPeerStatusCmd = "list status of peers" - helpPoolListCmd = "list all the nodes in the pool (including localhost)" + helpPeerListCmd = "list all the nodes in the pool (including localhost)" ) var ( - // Peer Detach Command Flags - flagPeerDetachForce bool + // Peer Remove Command Flags + flagPeerRemoveForce bool ) func init() { - peerCmd.AddCommand(peerProbeCmd) + peerCmd.AddCommand(peerAddCmd) - peerDetachCmd.Flags().BoolVarP(&flagPeerDetachForce, "force", "f", false, "Force") + peerRemoveCmd.Flags().BoolVarP(&flagPeerRemoveForce, "force", "f", false, "Force") - peerCmd.AddCommand(peerDetachCmd) + peerCmd.AddCommand(peerRemoveCmd) peerCmd.AddCommand(peerStatusCmd) - poolCmd.AddCommand(poolListCmd) + peerCmd.AddCommand(peerListCmd) RootCmd.AddCommand(peerCmd) - RootCmd.AddCommand(poolCmd) } var peerCmd = &cobra.Command{ @@ -45,28 +44,23 @@ var peerCmd = &cobra.Command{ Short: helpPeerCmd, } -var poolCmd = &cobra.Command{ - Use: "pool", - Short: helpPeerCmd, -} - -var peerProbeCmd = &cobra.Command{ - Use: "probe ", - Short: helpPeerProbeCmd, +var peerAddCmd = &cobra.Command{ + Use: "add ", + Short: helpPeerAddCmd, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { hostname := cmd.Flags().Args()[0] - peer, err := client.PeerProbe(hostname) + peer, err := client.PeerAdd(hostname) if err != nil { if verbose { log.WithFields(log.Fields{ "host": hostname, "error": err.Error(), - }).Error("peer probe failed") + }).Error("peer add failed") } - failure("Peer probe failed", err, 1) + failure("Peer add failed", err, 1) } - fmt.Println("Peer probe successful") + fmt.Println("Peer add successful") table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"ID", "Name", "Peer Addresses"}) table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, ",")}) @@ -74,26 +68,26 @@ var peerProbeCmd = &cobra.Command{ }, } -var peerDetachCmd = &cobra.Command{ - Use: "detach ", - Short: helpPeerDetachCmd, +var peerRemoveCmd = &cobra.Command{ + Use: "remove ", + Short: helpPeerRemoveCmd, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { hostname := cmd.Flags().Args()[0] peerID, err := getPeerID(hostname) if err == nil { - err = client.PeerDetach(peerID) + err = client.PeerRemove(peerID) } if err != nil { if verbose { log.WithFields(log.Fields{ "host": hostname, "error": err.Error(), - }).Error("peer detach failed") + }).Error("peer remove failed") } - failure("Peer detach failed", err, 1) + failure("Peer remove failed", err, 1) } - fmt.Println("Peer detach success") + fmt.Println("Peer remove success") }, } @@ -108,9 +102,9 @@ func peerStatusHandler(cmd *cobra.Command) { failure("Failed to get Peers list", err, 1) } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"ID", "Name", "Peer Addresses"}) + table.SetHeader([]string{"ID", "Name", "Peer Addresses", "Online"}) for _, peer := range peers { - table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, ",")}) + table.Append([]string{peer.ID.String(), peer.Name, strings.Join(peer.PeerAddresses, ","), formatBoolYesNo(peer.Online)}) } table.Render() } @@ -124,9 +118,9 @@ var peerStatusCmd = &cobra.Command{ }, } -var poolListCmd = &cobra.Command{ +var peerListCmd = &cobra.Command{ Use: "list", - Short: helpPoolListCmd, + Short: helpPeerListCmd, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { peerStatusHandler(cmd) diff --git a/glustercli/cmd/utils.go b/glustercli/cmd/utils.go new file mode 100644 index 000000000..f06ca3da7 --- /dev/null +++ b/glustercli/cmd/utils.go @@ -0,0 +1,8 @@ +package cmd + +func formatBoolYesNo(value bool) string { + if value == true { + return "yes" + } + return "no" +} diff --git a/glustercli/cmd/volume.go b/glustercli/cmd/volume.go index f255bbc67..5be764940 100644 --- a/glustercli/cmd/volume.go +++ b/glustercli/cmd/volume.go @@ -41,6 +41,10 @@ var ( flagExpandCmdReplicaCount int flagExpandCmdForce bool + // Filter Volume Info/List command flags + flagCmdFilterKey string + flagCmdFilterValue string + // Edit Command Flags flagCmdMetadataKey string flagCmdMetadataValue string @@ -62,10 +66,14 @@ func init() { volumeCmd.AddCommand(volumeGetCmd) volumeCmd.AddCommand(volumeResetCmd) + volumeInfoCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter by metadata key") + volumeInfoCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter by metadata value") volumeCmd.AddCommand(volumeInfoCmd) volumeCmd.AddCommand(volumeStatusCmd) + volumeListCmd.Flags().StringVar(&flagCmdFilterKey, "key", "", "Filter by metadata Key") + volumeListCmd.Flags().StringVar(&flagCmdFilterValue, "value", "", "Filter by metadata value") volumeCmd.AddCommand(volumeListCmd) // Volume Expand @@ -254,13 +262,16 @@ func volumeInfoDisplayNumbricks(vol api.VolumeGetResp) { } func volumeInfoDisplay(vol api.VolumeGetResp) { - fmt.Println() fmt.Println("Volume Name:", vol.Name) fmt.Println("Type:", vol.Type) fmt.Println("Volume ID:", vol.ID) fmt.Println("State:", vol.State) fmt.Println("Transport-type:", vol.Transport) + fmt.Println("Options:") + for key, value := range vol.Options { + fmt.Printf(" %s: %s\n", key, value) + } volumeInfoDisplayNumbricks(vol) for sIdx, subvol := range vol.Subvols { for bIdx, brick := range subvol.Bricks { @@ -281,8 +292,21 @@ func volumeInfoHandler2(cmd *cobra.Command, isInfo bool) error { volname = cmd.Flags().Args()[0] } if volname == "" { - vols, err = client.Volumes("") + if flagCmdFilterKey == "" && flagCmdFilterValue == "" { + vols, err = client.Volumes("") + } else if flagCmdFilterKey != "" && flagCmdFilterValue == "" { + vols, err = client.Volumes("", map[string]string{"key": flagCmdFilterKey}) + } else if flagCmdFilterKey == "" && flagCmdFilterValue != "" { + vols, err = client.Volumes("", map[string]string{"value": flagCmdFilterValue}) + } else if flagCmdFilterKey != "" && flagCmdFilterValue != "" { + vols, err = client.Volumes("", map[string]string{"key": flagCmdFilterKey, + "value": flagCmdFilterValue, + }) + } } else { + if flagCmdFilterKey != "" || flagCmdFilterValue != "" { + return errors.New("Invalid command. Cannot give filter arguments when providing volname") + } vols, err = client.Volumes(volname) } @@ -306,7 +330,7 @@ func volumeInfoHandler2(cmd *cobra.Command, isInfo bool) error { } var volumeInfoCmd = &cobra.Command{ - Use: "info", + Use: "info [ |--key |--value |--key --value ]", Short: helpVolumeInfoCmd, Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { @@ -323,7 +347,7 @@ var volumeInfoCmd = &cobra.Command{ } var volumeListCmd = &cobra.Command{ - Use: "list", + Use: "list [--key |--value |--key --value ]", Short: helpVolumeListCmd, Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { @@ -455,6 +479,6 @@ var volumeEditCmd = &cobra.Command{ } failure("Failed to edit metadata", err, 1) } - fmt.Printf("Metadata edit successfull\n") + fmt.Printf("Metadata edit successful\n") }, } diff --git a/glusterd2/commands/peers/addpeer.go b/glusterd2/commands/peers/addpeer.go index d5b11b24b..381dfef81 100644 --- a/glusterd2/commands/peers/addpeer.go +++ b/glusterd2/commands/peers/addpeer.go @@ -28,7 +28,7 @@ func addPeerHandler(w http.ResponseWriter, r *http.Request) { for key := range req.Metadata { if strings.HasPrefix(key, "_") { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, errors.ErrRestrictedKeyFound) + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrRestrictedKeyFound) return } } diff --git a/glusterd2/commands/volumes/commands.go b/glusterd2/commands/volumes/commands.go index 0b87b4b44..c8359b636 100644 --- a/glusterd2/commands/volumes/commands.go +++ b/glusterd2/commands/volumes/commands.go @@ -134,6 +134,12 @@ func (c *Command) Routes() route.Routes { Pattern: "/volfiles", Version: 1, HandlerFunc: volfilesListHandler}, + route.Route{ + Name: "VolfilesGet", + Method: "GET", + Pattern: "/volfiles/{volfileid:.*}", + Version: 1, + HandlerFunc: volfileGetHandler}, route.Route{ Name: "EditVolume", Method: "POST", diff --git a/glusterd2/commands/volumes/volfiles.go b/glusterd2/commands/volumes/volfiles.go index 3055f4305..0fda25f8c 100644 --- a/glusterd2/commands/volumes/volfiles.go +++ b/glusterd2/commands/volumes/volfiles.go @@ -1,10 +1,14 @@ package volumecommands import ( + "fmt" "net/http" restutils "github.com/gluster/glusterd2/glusterd2/servers/rest/utils" volgen "github.com/gluster/glusterd2/glusterd2/volgen2" + "github.com/gluster/glusterd2/pkg/errors" + + "github.com/gorilla/mux" ) func volfilesGenerateHandler(w http.ResponseWriter, r *http.Request) { @@ -33,3 +37,23 @@ func volfilesListHandler(w http.ResponseWriter, r *http.Request) { } restutils.SendHTTPResponse(ctx, w, http.StatusOK, volfiles) } + +func volfileGetHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + volfileid := mux.Vars(r)["volfileid"] + + volfile, err := volgen.GetVolfile(volfileid) + + if err != nil { + if err == errors.ErrVolFileNotFound { + restutils.SendHTTPError(ctx, w, http.StatusNotFound, errors.ErrVolFileNotFound) + } else { + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, fmt.Sprintf("unable to fetch volfile content %s", volfile)) + } + return + } + + restutils.SendHTTPResponse(ctx, w, http.StatusOK, nil) + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") + w.Write(volfile) +} diff --git a/glusterd2/commands/volumes/volume-create-txn.go b/glusterd2/commands/volumes/volume-create-txn.go index 0aa758cdb..301861cc4 100644 --- a/glusterd2/commands/volumes/volume-create-txn.go +++ b/glusterd2/commands/volumes/volume-create-txn.go @@ -197,9 +197,7 @@ func createVolinfo(c transaction.TxnCtx) error { checks.IsOnRoot = true } - if err := c.Set("brick-checks", &checks); err != nil { - return err - } + err = c.Set("brick-checks", &checks) - return nil + return err } diff --git a/glusterd2/commands/volumes/volume-expand-txn.go b/glusterd2/commands/volumes/volume-expand-txn.go index 0ca4e9250..07b4078e9 100644 --- a/glusterd2/commands/volumes/volume-expand-txn.go +++ b/glusterd2/commands/volumes/volume-expand-txn.go @@ -73,11 +73,9 @@ func expandValidatePrepare(c transaction.TxnCtx) error { return err } - if err := c.Set("volinfo", volinfo); err != nil { - return err - } + err = c.Set("volinfo", volinfo) - return nil + return err } func startBricksOnExpand(c transaction.TxnCtx) error { diff --git a/glusterd2/commands/volumes/volume-list.go b/glusterd2/commands/volumes/volume-list.go index 4b84a5455..06deecda7 100644 --- a/glusterd2/commands/volumes/volume-list.go +++ b/glusterd2/commands/volumes/volume-list.go @@ -11,12 +11,19 @@ import ( func volumeListHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - - volumes, err := volume.GetVolumes() + keys, keyFound := r.URL.Query()["key"] + values, valueFound := r.URL.Query()["value"] + filterParams := make(map[string]string) + if keyFound { + filterParams["key"] = keys[0] + } + if valueFound { + filterParams["value"] = values[0] + } + volumes, err := volume.GetVolumes(filterParams) if err != nil { restutils.SendHTTPError(ctx, w, http.StatusNotFound, err) } - resp := createVolumeListResp(volumes) restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) } diff --git a/glusterd2/commands/volumes/volume-option.go b/glusterd2/commands/volumes/volume-option.go index 7312f188a..9088869ab 100644 --- a/glusterd2/commands/volumes/volume-option.go +++ b/glusterd2/commands/volumes/volume-option.go @@ -55,11 +55,9 @@ func optionSetValidate(c transaction.TxnCtx) error { volinfo.Options[k] = v } - if err := c.Set("volinfo", volinfo); err != nil { - return err - } + err = c.Set("volinfo", volinfo) - return nil + return err } type txnOpType uint8 diff --git a/glusterd2/config.go b/glusterd2/config.go index 3db4308ce..cf92a5e80 100644 --- a/glusterd2/config.go +++ b/glusterd2/config.go @@ -171,9 +171,7 @@ func initConfig(confFile string) error { config.BindPFlags(flag.CommandLine) // Finally initialize missing config with defaults - if err := setDefaults(); err != nil { - return err - } + err := setDefaults() - return nil + return err } diff --git a/glusterd2/servers/eventlistener/eventtypes.go b/glusterd2/servers/eventlistener/eventtypes.go index 30ee07dc0..8a1c1bedf 100644 --- a/glusterd2/servers/eventlistener/eventtypes.go +++ b/glusterd2/servers/eventlistener/eventtypes.go @@ -1,8 +1,8 @@ package eventlistener var eventtypes = []string{ - "PEER_ATTACH", - "PEER_DETACH", + "PEER_ADD", + "PEER_REMOVE", "VOLUME_CREATE", "VOLUME_START", "VOLUME_STOP", diff --git a/glusterd2/volgen2/store-utils.go b/glusterd2/volgen2/store-utils.go index 600454c22..775672a97 100644 --- a/glusterd2/volgen2/store-utils.go +++ b/glusterd2/volgen2/store-utils.go @@ -2,8 +2,10 @@ package volgen2 import ( "context" + "strings" "github.com/gluster/glusterd2/glusterd2/store" + "github.com/gluster/glusterd2/pkg/errors" "github.com/coreos/etcd/clientv3" ) @@ -29,8 +31,22 @@ func GetVolfiles() ([]string, error) { volfiles := make([]string, len(resp.Kvs)) for i, kv := range resp.Kvs { - volfiles[i] = string(kv.Key) + volFile := strings.TrimPrefix(string(kv.Key), "volfiles/") + volfiles[i] = volFile } return volfiles, nil } + +//GetVolfile return particular volfile info +func GetVolfile(volfileID string) ([]byte, error) { + volfile := volfilePrefix + volfileID + resp, e := store.Store.Get(context.TODO(), volfile, clientv3.WithPrefix()) + if e != nil { + return []byte{}, e + } + if len(resp.Kvs) == 0 { + return []byte{}, errors.ErrVolFileNotFound + } + return resp.Kvs[0].Value, nil +} diff --git a/glusterd2/volume/store-utils.go b/glusterd2/volume/store-utils.go index 609f7681c..92acb7313 100644 --- a/glusterd2/volume/store-utils.go +++ b/glusterd2/volume/store-utils.go @@ -16,6 +16,17 @@ const ( volumePrefix string = "volumes/" ) +// metadataFilter is a filter type +type metadataFilter uint32 + +// GetVolumes Filter Types +const ( + noKeyAndValue metadataFilter = iota + onlyKey + onlyValue + keyAndValue +) + var ( // AddOrUpdateVolumeFunc marshals to volume object and passes to store to add/update AddOrUpdateVolumeFunc = AddOrUpdateVolume @@ -91,17 +102,38 @@ func GetVolumesList() (map[string]uuid.UUID, error) { return volumes, nil } +// getFilterType return the filter type for volume list/info +func getFilterType(filterParams map[string]string) metadataFilter { + _, key := filterParams["key"] + _, value := filterParams["value"] + if key && !value { + return onlyKey + } else if value && !key { + return onlyValue + } else if value && key { + return keyAndValue + } + return noKeyAndValue +} + //GetVolumes retrives the json objects from the store and converts them into //respective volinfo objects -func GetVolumes() ([]*Volinfo, error) { +func GetVolumes(filterParams ...map[string]string) ([]*Volinfo, error) { resp, e := store.Store.Get(context.TODO(), volumePrefix, clientv3.WithPrefix()) if e != nil { return nil, e } - volumes := make([]*Volinfo, len(resp.Kvs)) + var filterType metadataFilter + if len(filterParams) == 0 { + filterType = noKeyAndValue + } else { + filterType = getFilterType(filterParams[0]) + } + + var volumes []*Volinfo - for i, kv := range resp.Kvs { + for _, kv := range resp.Kvs { var vol Volinfo if err := json.Unmarshal(kv.Value, &vol); err != nil { @@ -111,8 +143,28 @@ func GetVolumes() ([]*Volinfo, error) { }).Error("Failed to unmarshal volume") continue } + switch filterType { + + case onlyKey: + if _, keyFound := vol.Metadata[filterParams[0]["key"]]; keyFound { + volumes = append(volumes, &vol) + } + case onlyValue: + for _, value := range vol.Metadata { + if value == filterParams[0]["value"] { + volumes = append(volumes, &vol) + } + } + case keyAndValue: + if value, keyFound := vol.Metadata[filterParams[0]["key"]]; keyFound { + if value == filterParams[0]["value"] { + volumes = append(volumes, &vol) + } + } + default: + volumes = append(volumes, &vol) - volumes[i] = &vol + } } return volumes, nil diff --git a/pkg/api/volume_resp.go b/pkg/api/volume_resp.go index e9212d0d8..b148ed202 100644 --- a/pkg/api/volume_resp.go +++ b/pkg/api/volume_resp.go @@ -76,6 +76,16 @@ type VolumeStatusResp struct { type VolumeCreateResp VolumeInfo // VolumeGetResp is the response sent for a volume get request. +/* VolumeGetResp can also be filtered based on query parameters +sent along with volume list/info api. +Query Parameters can be either metadata key/value/both key and value. +Example of API request- + 1- /volumes - + a- GET 'http://localhost:24007/v1/volumes?key={keyname}&value={value} + b- GET 'http://localhost:24007/v1/volumes?key={keyname}' + c- GET 'http://localhost:24007/v1/volumes?value={value}' +Note - Cannot use query parameters or cli flags if volname is also supplied. +*/ type VolumeGetResp VolumeInfo // VolumeExpandResp is the response sent for a volume expand request. @@ -91,6 +101,16 @@ type VolumeStopResp VolumeInfo type VolumeOptionResp VolumeInfo // VolumeListResp is the response sent for a volume list request. +/* VolumeListResp can also be filtered based on query parameters +sent along with volume list/info api. +Query Parameters can be either metadata key/value/both key and value. +Example of API request- + 1- /volumes - + a- GET 'http://localhost:24007/v1/volumes?key={keyname}&value={value} + b- GET 'http://localhost:24007/v1/volumes?key={keyname}' + c- GET 'http://localhost:24007/v1/volumes?value={value}' +Note - Cannot use query parameters or cli flags if volname is also supplied. +*/ type VolumeListResp []VolumeGetResp // OptionGroupListResp is the response sent for a group list request. diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 50020f776..95a175087 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -41,4 +41,5 @@ var ( ErrClusterNotFound = errors.New("Cluster instance not found in store") ErrDuplicateBrickPath = errors.New("Duplicate brick entry") ErrRestrictedKeyFound = errors.New("Key names starting with '_' are restricted in metadata field") + ErrVolFileNotFound = errors.New("volume file not found") ) diff --git a/pkg/restclient/common.go b/pkg/restclient/common.go index 949881cc8..0a368bf79 100644 --- a/pkg/restclient/common.go +++ b/pkg/restclient/common.go @@ -16,8 +16,9 @@ import ( "github.com/dgrijalva/jwt-go" ) -var ( +const ( expireSeconds = 120 + clientTimeout = 30 ) // Client represents Glusterd2 REST Client @@ -47,7 +48,7 @@ func parseHTTPError(jsonData []byte) string { func getAuthToken(username string, password string) string { // Create the Claims claims := &jwt.StandardClaims{ - ExpiresAt: time.Now().Add(time.Second * time.Duration(expireSeconds)).Unix(), + ExpiresAt: time.Now().Add(time.Second * expireSeconds).Unix(), Issuer: username, } token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) @@ -93,6 +94,7 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod } req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") + req.Close = true // Set Authorization if username and password is not empty string if c.username != "" && c.password != "" { @@ -100,9 +102,8 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod } tr := &http.Transport{ - DisableCompression: true, - DisableKeepAlives: true, - ResponseHeaderTimeout: 3 * time.Second, + DisableCompression: true, + DisableKeepAlives: true, } if c.cacert != "" || c.insecure { @@ -120,7 +121,9 @@ func (c *Client) do(method string, url string, data interface{}, expectStatusCod } } - client := &http.Client{Transport: tr} + client := &http.Client{ + Transport: tr, + Timeout: clientTimeout * time.Second} resp, err := client.Do(req) if err != nil { diff --git a/pkg/restclient/peer.go b/pkg/restclient/peer.go index 37281c401..2cf301780 100644 --- a/pkg/restclient/peer.go +++ b/pkg/restclient/peer.go @@ -7,8 +7,8 @@ import ( "github.com/gluster/glusterd2/pkg/api" ) -// PeerProbe adds a peer to the Cluster -func (c *Client) PeerProbe(host string) (api.PeerAddResp, error) { +// PeerAdd adds a peer to the Cluster +func (c *Client) PeerAdd(host string) (api.PeerAddResp, error) { peerAddReq := api.PeerAddReq{ Addresses: []string{host}, @@ -19,8 +19,8 @@ func (c *Client) PeerProbe(host string) (api.PeerAddResp, error) { return resp, err } -// PeerDetach detaches a peer from the Cluster -func (c *Client) PeerDetach(peerid string) error { +// PeerRemove removes a peer from the Cluster +func (c *Client) PeerRemove(peerid string) error { delURL := fmt.Sprintf("/v1/peers/%s", peerid) return c.del(delURL, nil, http.StatusNoContent, nil) } diff --git a/pkg/restclient/volume.go b/pkg/restclient/volume.go index 568ab5bb2..8b3b449dd 100644 --- a/pkg/restclient/volume.go +++ b/pkg/restclient/volume.go @@ -3,10 +3,22 @@ package restclient import ( "fmt" "net/http" + "net/url" "github.com/gluster/glusterd2/pkg/api" ) +// metadataFilter is a filter type +type metadataFilter uint32 + +// GetVolumes Filter Types +const ( + noKeyAndValue metadataFilter = iota + onlyKey + onlyValue + keyAndValue +) + // VolumeCreate creates Gluster Volume func (c *Client) VolumeCreate(req api.VolCreateReq) (api.VolumeCreateResp, error) { var vol api.VolumeCreateResp @@ -14,11 +26,44 @@ func (c *Client) VolumeCreate(req api.VolCreateReq) (api.VolumeCreateResp, error return vol, err } +// getFilterType return the filter type for volume list/info +func getFilterType(filterParams map[string]string) metadataFilter { + _, key := filterParams["key"] + _, value := filterParams["value"] + if key && !value { + return onlyKey + } else if value && !key { + return onlyValue + } else if value && key { + return keyAndValue + } + return noKeyAndValue +} + +// getQueryString returns the query string for filtering volumes +func getQueryString(filterParam map[string]string) string { + filterType := getFilterType(filterParam) + var queryString string + switch filterType { + case onlyKey: + queryString = fmt.Sprintf("?key=%s", url.QueryEscape(filterParam["key"])) + case onlyValue: + queryString = fmt.Sprintf("?value=%s", url.QueryEscape(filterParam["value"])) + case keyAndValue: + queryString = fmt.Sprintf("?key=%s&value=%s", url.QueryEscape(filterParam["key"]), url.QueryEscape(filterParam["value"])) + } + return queryString +} + // Volumes returns list of all volumes -func (c *Client) Volumes(volname string) (api.VolumeListResp, error) { +func (c *Client) Volumes(volname string, filterParams ...map[string]string) (api.VolumeListResp, error) { if volname == "" { var vols api.VolumeListResp - url := fmt.Sprintf("/v1/volumes") + var queryString string + if len(filterParams) > 0 { + queryString = getQueryString(filterParams[0]) + } + url := fmt.Sprintf("/v1/volumes%s", queryString) err := c.get(url, nil, http.StatusOK, &vols) return vols, err } diff --git a/pkg/utils/peerutils_test.go b/pkg/utils/peerutils_test.go index cf900cc09..efd5a449b 100644 --- a/pkg/utils/peerutils_test.go +++ b/pkg/utils/peerutils_test.go @@ -16,14 +16,14 @@ func TestIsPeerAddressSame(t *testing.T) { } func TestFormRemotePeerAddress(t *testing.T) { - peer, err := FormRemotePeerAddress("192.168.1.1:8080") + _, err := FormRemotePeerAddress("192.168.1.1:8080") assert.Nil(t, err) config.SetDefault("defaultpeerport", "80") - peer, err = FormRemotePeerAddress("192.168.1.1") + peer, err := FormRemotePeerAddress("192.168.1.1") assert.Equal(t, peer, "192.168.1.1:80") - peer, err = FormRemotePeerAddress(":8080") + _, err = FormRemotePeerAddress(":8080") assert.Contains(t, err.Error(), "Invalid peer address") } diff --git a/plugins/device/api/req.go b/plugins/device/api/req.go index 29fabc803..75e0458f9 100644 --- a/plugins/device/api/req.go +++ b/plugins/device/api/req.go @@ -6,12 +6,9 @@ const ( // DeviceDisabled represents disabled DeviceDisabled = "Disabled" - - // DeviceFailed represents failed - DeviceFailed = "Failed" ) // AddDeviceReq structure type AddDeviceReq struct { - Devices []string `json:"devices"` + Device string `json:"device"` } diff --git a/plugins/device/cmdexec/device.go b/plugins/device/cmdexec/device.go deleted file mode 100644 index 6c4dedfae..000000000 --- a/plugins/device/cmdexec/device.go +++ /dev/null @@ -1,51 +0,0 @@ -package cmdexec - -import ( - "os/exec" - "strings" - - "github.com/gluster/glusterd2/glusterd2/transaction" -) - -func createVgName(device string) string { - vgName := strings.Replace("vg"+device, "/", "-", -1) - return vgName -} - -// DeviceSetup is used to prepare device before using devices. -func DeviceSetup(c transaction.TxnCtx, device string) error { - - var err error - defer func() { - if err != nil { - DeviceCleanup(c, device) - } - }() - pvcreateCmd := exec.Command("pvcreate", "--metadatasize=128M", "--dataalignment=256K", device) - if err := pvcreateCmd.Run(); err != nil { - c.Logger().WithError(err).WithField("device", device).Error("pvcreate failed for device") - return err - } - vgName := createVgName(device) - vgcreateCmd := exec.Command("vgcreate", vgName, device) - if err = vgcreateCmd.Run(); err != nil { - c.Logger().WithError(err).WithField("device", device).Error("vgcreate failed for device") - return err - } - - return nil - -} - -// DeviceCleanup is used to clean up devices. -func DeviceCleanup(c transaction.TxnCtx, device string) { - vgName := createVgName(device) - vgremoveCmd := exec.Command("vgremove", vgName) - if err := vgremoveCmd.Run(); err != nil { - c.Logger().WithError(err).WithField("device", device).Error("vgremove failed for device") - } - pvremoveCmd := exec.Command("pvremove", device) - if err := pvremoveCmd.Run(); err != nil { - c.Logger().WithError(err).WithField("device", device).Error("pvremove failed for device") - } -} diff --git a/plugins/device/deviceutils/utils.go b/plugins/device/deviceutils/utils.go index 105259d4b..1df5cf538 100644 --- a/plugins/device/deviceutils/utils.go +++ b/plugins/device/deviceutils/utils.go @@ -4,9 +4,8 @@ import ( "os/exec" ) -// +//CreatePV is used to create physical volume. func CreatePV(device string) error { - pvcreateCmd := exec.Command("pvcreate", "--metadatasize=128M", "--dataalignment=256K", device) if err := pvcreateCmd.Run(); err != nil { return err @@ -14,9 +13,8 @@ func CreatePV(device string) error { return nil } -// +//CreateVG is used to create volume group func CreateVG(device string, vgName string) error { - vgcreateCmd := exec.Command("vgcreate", vgName, device) if err := vgcreateCmd.Run(); err != nil { return err @@ -24,7 +22,7 @@ func CreateVG(device string, vgName string) error { return nil } -// +//RemoveVG is used to remove volume group. func RemoveVG(vgName string) error { vgremoveCmd := exec.Command("vgremove", vgName) if err := vgremoveCmd.Run(); err != nil { @@ -33,7 +31,7 @@ func RemoveVG(vgName string) error { return nil } -// +//RemovePV is used to remove physical volume func RemovePV(device string) error { pvremoveCmd := exec.Command("pvremove", device) if err := pvremoveCmd.Run(); err != nil { diff --git a/plugins/device/init.go b/plugins/device/init.go index 21966c90f..8233ab97e 100644 --- a/plugins/device/init.go +++ b/plugins/device/init.go @@ -26,8 +26,7 @@ func (p *Plugin) RestRoutes() route.Routes { Version: 1, RequestType: utils.GetTypeString((*deviceapi.AddDeviceReq)(nil)), ResponseType: utils.GetTypeString((*deviceapi.AddDeviceResp)(nil)), - HandlerFunc: deviceAddHandler, - }, + HandlerFunc: deviceAddHandler}, } } diff --git a/plugins/device/rest.go b/plugins/device/rest.go index 83128a452..16cf9e9a4 100644 --- a/plugins/device/rest.go +++ b/plugins/device/rest.go @@ -49,7 +49,7 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { if err == errors.ErrPeerNotFound { restutils.SendHTTPError(ctx, w, http.StatusNotFound, errors.ErrPeerNotFound) } else { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Failed to get peer from store") + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, "Failed to get peer details from store") } return } @@ -61,9 +61,9 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - if !CheckIfDeviceExist(req.Devices, devices) { - logger.WithError(err).WithField("device", req.Devices).Error(" One or more devices already exists") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, " One or more devices already exists") + if !CheckIfDeviceExist(req.Device, devices) { + logger.WithError(err).WithField("device", req.Device).Error("Device already exists") + restutils.SendHTTPError(ctx, w, http.StatusBadRequest, "Device already exists") return } @@ -83,12 +83,13 @@ func deviceAddHandler(w http.ResponseWriter, r *http.Request) { restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } - err = txn.Ctx.Set("", &req.Devices) + err = txn.Ctx.Set("device", &req.Device) if err != nil { - logger.WithError(err).WithField("key", "").Error("Failed to set key in transaction context") + logger.WithError(err).WithField("key", "device").Error("Failed to set key in transaction context") restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } + err = txn.Do() if err != nil { logger.WithError(err).Error("Transaction to prepare device failed") diff --git a/plugins/device/store-utils.go b/plugins/device/store-utils.go index 8ea4e4e46..9d00d54fe 100644 --- a/plugins/device/store-utils.go +++ b/plugins/device/store-utils.go @@ -24,20 +24,17 @@ func GetDevices(peerID string) ([]deviceapi.Info, error) { } //CheckIfDeviceExist returns error if all devices already exist or returns list of devices to be added -func CheckIfDeviceExist(reqDevices []string, devices []deviceapi.Info) bool { - - for _, key := range reqDevices { - for _, reqKey := range devices { - if key == reqKey.Name { - return false - } +func CheckIfDeviceExist(reqDevice string, devices []deviceapi.Info) bool { + for _, key := range devices { + if reqDevice == key.Name { + return false } } return true } -// AddDevices adds device to specific peer -func AddDevices(devices []deviceapi.Info, peerID string) error { +// AddDevice adds device of specific peer +func AddDevice(device deviceapi.Info, peerID string) error { deviceDetails, err := GetDevices(peerID) if err != nil { return err @@ -46,8 +43,9 @@ func AddDevices(devices []deviceapi.Info, peerID string) error { if err != nil { return err } + var devices []deviceapi.Info if deviceDetails != nil { - devices = append(devices, deviceDetails...) + devices = append(deviceDetails, device) } deviceJSON, err := json.Marshal(devices) if err != nil { @@ -60,5 +58,4 @@ func AddDevices(devices []deviceapi.Info, peerID string) error { } return nil - } diff --git a/plugins/device/transaction.go b/plugins/device/transaction.go index e6d743c63..c5628346a 100644 --- a/plugins/device/transaction.go +++ b/plugins/device/transaction.go @@ -15,46 +15,36 @@ func txnPrepareDevice(c transaction.TxnCtx) error { return err } - var devices []string - if err := c.Get("devices", &devices); err != nil { - c.Logger().WithError(err).WithField("key", "req").Error("Failed to get key from transaction context") + var device string + if err := c.Get("device", &device); err != nil { + c.Logger().WithError(err).WithField("key", "device").Error("Failed to get key from transaction context") return err } - var deviceList []deviceapi.Info - for _, name := range devices { - tempDevice := deviceapi.Info{ - Name: name, - } - deviceList = append(deviceList, tempDevice) - } + var deviceInfo deviceapi.Info + deviceInfo.Name = device - var failedDevice []string - var successDevice []deviceapi.Info - for index, device := range deviceList { - err := deviceutils.CreatePV(device.Name) - if err != nil { - c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to create physical volume") - continue - } - vgName := strings.Replace("vg"+device.Name, "/", "-", -1) - err = deviceutils.CreateVG(device.Name, vgName) - if err != nil { - c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to create volume group") - err = deviceutils.RemovePV(device.Name) - if err != nil { - c.Logger().WithError(err).WithField("device", device.Name).Error("Failed to remove physical volume") - failedDevice = append(failedDevice, device.Name) - } + err := deviceutils.CreatePV(device) + if err != nil { + c.Logger().WithError(err).WithField("device", device).Error("Failed to create physical volume") + return err + } + vgName := strings.Replace("vg"+device, "/", "-", -1) + err = deviceutils.CreateVG(device, vgName) + if err != nil { + c.Logger().WithError(err).WithField("device", device).Error("Failed to create volume group") + err_PV := deviceutils.RemovePV(device) + if err_PV != nil { + c.Logger().WithError(err).WithField("device", device).Error("Failed to remove physical volume") } - c.Logger().WithError(err).WithField("device", device.Name).Error("Setup device successful, setting device status to 'DeviceEnabled'") - deviceList[index].State = deviceapi.DeviceEnabled - successDevice = append(successDevice, deviceList[index]) + return err } + c.Logger().WithField("device", device).Info("Device setup successful, setting device status to 'DeviceEnabled'") + deviceInfo.State = deviceapi.DeviceEnabled - err := AddDevices(successDevice, peerID) + err = AddDevice(deviceInfo, peerID) if err != nil { - c.Logger().WithError(err).Error("Couldn't add deviceinfo to store") + c.Logger().WithError(err).WithField("peerid", peerID).Error("Couldn't add deviceinfo to store") return err } return nil