pmap: Synchronize port allocation across instances

This change synchronizes port allocation on a node/machine when multiple
glusterd2 instances are running (e2e tests). This solution is to be
treated as stopgap fix. A long term and more robust solution is to let
the bricks pick and bind on their own.

Signed-off-by: Prashanth Pai <ppai@redhat.com>

glusterd2/pmap/pmap.go

@@ -6,6 +6,9 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -37,6 +40,8 @@ type registryType struct {
 	BasePort     int                       `json:"base_port"`
 	LastAlloc    int                       `json:"last_allocated_port,omitempty"`
 	Ports        [gfPortMax + 1]portStatus `json:"ports,omitempty"`
+
+	portLockFds map[int]int
 }
 
 func (r *registryType) String() string {
@@ -65,6 +70,56 @@ func (r *registryType) MarshalJSON() ([]byte, error) {
 	})
 }
 
+// This change synchronizes port allocation on a node/machine when multiple
+// glusterd2 instances are running (e2e tests). This solution is to be
+// treated as stopgap fix. A long term and more robust solution is to let
+// the bricks pick and bind on their own.
+
+func (r *registryType) TryPortLock(port int) bool {
+
+	if _, ok := r.portLockFds[port]; ok {
+		// we already have a lock
+		return false
+	}
+
+	portLockFile := fmt.Sprintf("/var/run/glusterd2_pmap_port_%d.lock", port)
+	fd, err := syscall.Open(portLockFile,
+		syscall.O_CREAT|syscall.O_WRONLY|syscall.O_CLOEXEC, 0666)
+	if err != nil {
+		return false
+	}
+
+	err = syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB)
+	switch err {
+	case nil:
+		// keep the fd open if we get the lock, close otherwise
+		r.portLockFds[port] = fd
+		log.WithField("lock-file", portLockFile).Debug(
+			"Obtained lock on pmap port lock file.")
+		return true
+	case syscall.EWOULDBLOCK:
+		log.WithField("lock-file", portLockFile).Debug(
+			"Failed to obtain lock on pmap port lock file.")
+	}
+
+	syscall.Close(fd)
+	return false
+}
+
+func (r *registryType) PortUnlock(port int) {
+	fd, ok := r.portLockFds[port]
+	if !ok {
+		return
+	}
+	syscall.Flock(fd, syscall.LOCK_UN)
+	syscall.Close(fd)
+	delete(r.portLockFds, port)
+	if registry.Ports[port].Type == GfPmapPortFree || registry.Ports[port].Type == GfPmapPortForeign {
+		portLockFile := fmt.Sprintf("/var/run/glusterd2_pmap_port_%d.lock", port)
+		syscall.Unlink(portLockFile)
+	}
+}
+
 var registry = new(registryType)
 
 func isPortFree(port int) bool {
@@ -133,14 +188,18 @@ func registryAlloc(recheckForeign bool) int {
 		if registry.Ports[p].Type == GfPmapPortFree ||
 			(recheckForeign && registry.Ports[p].Type == GfPmapPortForeign) {
 
-			if isPortFree(p) {
-				registry.Ports[p].Type = GfPmapPortLeased
-				port = p
-				break
-			} else {
-				// We may have an opportunity here to change
-				// the port's status from GfPmapPortFree to
-				// GfPmapPortForeign. Passing on for now...
+			if registry.TryPortLock(p) {
+				if isPortFree(p) {
+					registry.Ports[p].Type = GfPmapPortLeased
+					port = p
+					// keep port file locked if port was
+					// found to be free and we leased it to
+					// a brick
+					break
+				} else {
+					registry.Ports[p].Type = GfPmapPortForeign
+					registry.PortUnlock(p)
+				}
 			}
 		}
 	}
@@ -223,6 +282,7 @@ func doRemove(port int, brickname string, xprt interface{}) {
 			registry.Ports[port].Xprt = nil
 		}
 	}
+	registry.PortUnlock(port)
 }
 
 func registryRemove(port int, brickname string, ptype PortType, xprt interface{}) {
@@ -254,6 +314,8 @@ func initRegistry() {
 	registry.Lock()
 	defer registry.Unlock()
 
+	registry.portLockFds = make(map[int]int)
+
 	// TODO: When a config option by the name 'base-port'
 	// becomes available, use that.
 	registry.BasePort = gfIanaPrivPortsStart