glusterd2: Support setting Peer and Cluster IDs from the environment

This is an extension to b8f9625, which makes it much easier to provision
and deploy GD2 clusters in a K8S.

With this patch GD2 reads GD2_PEER_ID and GD2_CLUSTER_ID from the
environment if available, and saved into LOCALSTATEDIR/uuid.toml.
If both environment variables and the uuid file have IDs, preference is
given to the environment variables, over the uuid file.

Closes #1113

glusterd2/gdctx/id.go

@@ -1,20 +1,33 @@
 package gdctx
 
 import (
+	"errors"
 	"expvar"
 	"io/ioutil"
 	"os"
 	"path"
+	"strings"
+	"sync"
 
 	"github.com/pborman/uuid"
 	toml "github.com/pelletier/go-toml"
-	log "github.com/sirupsen/logrus"
 	config "github.com/spf13/viper"
 )
 
+const (
+	uuidFileName    = "uuid.toml"
+	peerIDKey       = "peer-id"
+	clusterIDKey    = "cluster-id"
+	envPrefix       = "GD2"
+	envPeerIDKey    = envPrefix + "_PEER_ID"
+	envClusterIDKey = envPrefix + "_CLUSTER_ID"
+)
+
 var (
-	expPeerID    = expvar.NewString("peer-id")
-	expClusterID = expvar.NewString("cluster-id")
+	expPeerID    = expvar.NewString(peerIDKey)
+	expClusterID = expvar.NewString(clusterIDKey)
+
+	idMut sync.Mutex
 )
 
 var (
@@ -24,74 +37,112 @@ var (
 	MyClusterID uuid.UUID
 )
 
-const uuidFileName = "uuid.toml"
+func uuidFilePath() string {
+	return path.Join(config.GetString("localstatedir"), uuidFileName)
+}
 
-// UUIDConfig is a type that is read from and written to uuidFileName file.
-type UUIDConfig struct {
-	PeerID    string `toml:"peer-id"`
-	ClusterID string `toml:"cluster-id"`
+// uuidConfig is a type that gives the configured values for peer and cluster ids
+// from the following sources in order of preference
+// - environment variables (GD2_CLUSTER_ID and GD2_PEER_ID)
+// - the uuid config file ($LOCALSTATEDIR/uuid.toml)
+// - randomly generated uuid
+type uuidConfig struct {
+	*config.Viper
 }
 
-func (cfg *UUIDConfig) reload() error {
+func newUUIDConfig() *uuidConfig {
+	uc := &uuidConfig{config.New()}
 
-	uuidFilePath := path.Join(config.GetString("localstatedir"), uuidFileName)
-	b, err := ioutil.ReadFile(uuidFilePath)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	} else if err == nil {
-		if err := toml.Unmarshal(b, cfg); err != nil {
+	// First setup config to use environment variables
+	// TODO: Should be using a common prefix constant here and in the main
+	// glusterd2 package
+	uc.SetEnvPrefix(envPrefix)
+	uc.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	uc.AutomaticEnv()
+
+	// Next setup config to read in from the config file
+	uc.SetConfigFile(uuidFilePath())
+	uc.SetConfigType("toml")
+
+	// Finally set random uuids as the default
+	uc.SetDefault(peerIDKey, uuid.NewRandom().String())
+	uc.SetDefault(clusterIDKey, uuid.NewRandom().String())
+
+	return uc
+}
+
+func (uc *uuidConfig) reload(init bool) error {
+	// Reload config from file
+	if err := uc.ReadInConfig(); err != nil {
+		// Error out if not initializing
+		if !init {
 			return err
 		}
-	}
 
-	if cfg.PeerID == "" {
-		cfg.PeerID = uuid.New()
-		log.WithField("peer-id", cfg.PeerID).Info("Generated new peer ID")
+		// If initalizing, ignore ENOENT error
+		if !os.IsNotExist(err) {
+			return err
+		}
 	}
 
-	if cfg.ClusterID == "" {
-		cfg.ClusterID = uuid.New()
-		log.WithField("cluster-id", cfg.ClusterID).Info("Generated new cluster ID")
+	peerID := uc.GetString(peerIDKey)
+	clusterID := uc.GetString(clusterIDKey)
+
+	MyUUID = uuid.Parse(peerID)
+	if MyUUID == nil {
+		return errors.New("could not parse peer-id")
+	}
+	MyClusterID = uuid.Parse(clusterID)
+	if MyClusterID == nil {
+		return errors.New("could not parse cluster-id")
 	}
 
-	MyUUID = uuid.Parse(cfg.PeerID)
-	MyClusterID = uuid.Parse(cfg.ClusterID)
 	expPeerID.Set(MyUUID.String())
 	expClusterID.Set(MyClusterID.String())
 
 	return nil
 }
 
-func (cfg *UUIDConfig) save() error {
+func (uc *uuidConfig) save() error {
+	tmpCfg := struct {
+		PeerID    string `toml:"peer-id"`
+		ClusterID string `toml:"cluster-id"`
+	}{
+		PeerID:    uc.GetString(peerIDKey),
+		ClusterID: uc.GetString(clusterIDKey),
+	}
 
-	b, err := toml.Marshal(*cfg)
+	b, err := toml.Marshal(tmpCfg)
 	if err != nil {
 		return err
 	}
 
-	uuidFilePath := path.Join(config.GetString("localstatedir"), uuidFileName)
-	return ioutil.WriteFile(uuidFilePath, b, 0644)
+	return ioutil.WriteFile(uuidFilePath(), b, 0644)
 }
 
 // UpdateClusterID shall update the cluster ID and save it to file.
 func UpdateClusterID(id string) error {
-	cfg := &UUIDConfig{
-		PeerID:    MyUUID.String(),
-		ClusterID: id,
-	}
+	idMut.Lock()
+	defer idMut.Unlock()
+
+	cfg := newUUIDConfig()
+	cfg.Set(clusterIDKey, id)
 
 	if err := cfg.save(); err != nil {
 		return err
 	}
 
-	return cfg.reload()
+	return cfg.reload(false)
 }
 
-// InitUUID will generate (or use if present) node ID and cluster ID.
+// InitUUID intializes the peer and cluster IDs using the configured or saved
+// values if available, or with random uuids
 func InitUUID() error {
-	cfg := &UUIDConfig{}
+	idMut.Lock()
+	defer idMut.Unlock()
 
-	if err := cfg.reload(); err != nil {
+	cfg := newUUIDConfig()
+	if err := cfg.reload(true); err != nil {
 		return err
 	}
 

glusterd2/gdctx/id_test.go

@@ -1,41 +1,189 @@
 package gdctx
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 
+	"github.com/pborman/uuid"
 	config "github.com/spf13/viper"
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func TestInitUUID(t *testing.T) {
-	err := InitUUID()
-	assert.Nil(t, err)
+const (
+	invalidUUID = "this-is-an-invalid-uuid"
+)
+
+var (
+	testStateDir string
+)
+
+func resetEnv(t *testing.T) {
+	MyUUID = nil
+	MyClusterID = nil
+	require.NoError(t, os.Setenv(envClusterIDKey, ""))
+	require.NoError(t, os.Setenv(envPeerIDKey, ""))
+
+	require.NoError(t, os.RemoveAll(testStateDir))
+	require.NoError(t, os.MkdirAll(testStateDir, os.ModePerm))
+	config.Set("localstatedir", testStateDir)
+
+}
+
+// TestIDs tests various ways the peer and cluster ids can be loaded, updated and saved
+func TestIDs(t *testing.T) {
+	d, err := ioutil.TempDir("", "TestGdtxUUID")
+	require.NoError(t, err)
+	defer os.RemoveAll(d)
+
+	testStateDir = d
+
+	t.Run("Init", testInitUUID)
+	t.Run("SaveToFile", testSaveFile)
+	t.Run("ReloadFromFile", testReloadFile)
+	t.Run("LoadFromENV", testEnvIDs)
+	t.Run("UpdateClusterID", testUpdateClusterID)
+}
+
+// testInitUUID ensures that InitUUID properly sets MyUUID and MyClusterID
+func testInitUUID(t *testing.T) {
+	resetEnv(t)
+
+	// Empty run
+	require.NoError(t, InitUUID())
+
+	// Ensure that both uuids are not nil anymore
+	require.NotNil(t, MyUUID)
+	require.False(t, uuid.Equal(uuid.NIL, MyUUID))
+	require.NotNil(t, MyUUID)
+	require.False(t, uuid.Equal(uuid.NIL, MyClusterID))
+}
+
+// testUpdateClusterID ensures that UpdateClusterID properly updates and saves
+// the cluster-id to the given UUID
+func testUpdateClusterID(t *testing.T) {
+	resetEnv(t)
+
+	clusterID := uuid.NewRandom()
+	clusterIDstr := clusterID.String()
+
+	// Ensure that a valid uuid is set and updated
+	require.NoError(t, UpdateClusterID(clusterIDstr))
+	require.True(t, uuid.Equal(MyClusterID, clusterID))
+
+	// Ensure that an invalid uuid cannot be set
+	require.Error(t, UpdateClusterID("this-is-an-invalid-uuid"))
+}
+
+// TestSaveFile tests saving and reloading uuid from file
+func testSaveFile(t *testing.T) {
+	resetEnv(t)
+
+	// Storing the randomly initalized ids in a fresh uuidConfig and saving it file
+	c1 := newUUIDConfig()
+	peerID := c1.GetString(peerIDKey)
+	clusterID := c1.GetString(clusterIDKey)
 
-	config.Set("localstatedir", "/tmp/gd2test/test")
-	err = InitUUID()
-	assert.NotNil(t, err)
+	require.NoError(t, c1.save())
 
-	os.Remove("uuid.toml")
+	// Create a new uuidConfig that will load values from the saved file
+	c2 := newUUIDConfig()
+	require.NoError(t, c2.reload(false))
 
+	require.Equal(t, peerID, c2.GetString(peerIDKey))
+	require.Equal(t, clusterID, c2.GetString(clusterIDKey))
 }
 
-func TestSave(t *testing.T) {
-	defer os.Remove("uuid.toml")
-	config.Set("localstatedir", "")
-	cfg := &UUIDConfig{}
+// testReloadFile tests if reloading uuid file works correctly in different cases
+// NOTE: Successful case is being tested in testSaveFile
+func testReloadFile(t *testing.T) {
+	t.Run("NotPresent", testReloadFileNoFile)
+	t.Run("Empty", testReloadFileEmptyFile)
+	t.Run("InvalidTOML", testReloadFileInvalidTOML)
+	t.Run("InvalidUUID", testReloadFileInvalidUUID)
+}
+
+func testReloadFileNoFile(t *testing.T) {
+	resetEnv(t)
+
+	c1 := newUUIDConfig()
+
+	// Reloading the file should fail if it is missing, except during initialization
+	require.Error(t, c1.reload(false))
+	require.NoError(t, c1.reload(true))
+}
+
+func testReloadFileEmptyFile(t *testing.T) {
+	resetEnv(t)
+
+	f, err := os.Create(uuidFilePath())
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	c1 := newUUIDConfig()
+	// Reloading empty file should always succeed
+	require.NoError(t, c1.reload(false))
+	require.NoError(t, c1.reload(true))
+}
+
+func testReloadFileInvalidTOML(t *testing.T) {
+	resetEnv(t)
+
+	f, err := os.Create(uuidFilePath())
+	require.NoError(t, err)
+	_, err = f.WriteString("this: is: not: toml\nno really")
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	c1 := newUUIDConfig()
+	// Reloading file with invalid toml should always fail
+	require.Error(t, c1.reload(false))
+	require.Error(t, c1.reload(true))
+}
+
+func testReloadFileInvalidUUID(t *testing.T) {
+	resetEnv(t)
+
+	c1 := newUUIDConfig()
+	// Set invalid uuid as config and save to file
+	c1.Set(peerIDKey, invalidUUID)
+	c1.Set(clusterIDKey, invalidUUID)
+	require.NoError(t, c1.save())
+
+	c2 := newUUIDConfig()
+	// Reloading file with invalid uuids should fail always
+	require.Error(t, c2.reload(false))
+	require.Error(t, c2.reload(true))
+}
+
+// testEnvIDs tests getting the ids from the environment
+func testEnvIDs(t *testing.T) {
+	t.Run("ValidUUIDs", testEnvIDsValid)
+	t.Run("InvalidUUIDs", testEnvIDsInvalid)
+}
+
+func testEnvIDsValid(t *testing.T) {
+	resetEnv(t)
+
+	// Ensure that valid ids are loaded from the environment
+	peerID := uuid.NewRandom()
+	clusterID := uuid.NewRandom()
 
-	err := cfg.save()
-	assert.Nil(t, err)
+	require.NoError(t, os.Setenv(envPeerIDKey, peerID.String()))
+	require.NoError(t, os.Setenv(envClusterIDKey, clusterID.String()))
 
-	config.Set("localstatedir", "/tmp/gd2test/test")
-	err = cfg.save()
-	assert.NotNil(t, err)
+	require.NoError(t, InitUUID())
+	require.True(t, uuid.Equal(peerID, MyUUID))
+	require.True(t, uuid.Equal(clusterID, MyClusterID))
 }
 
-func TestReload(t *testing.T) {
-	cfg := &UUIDConfig{}
+func testEnvIDsInvalid(t *testing.T) {
+	// Ensure that invalid ids are not loaded
+	resetEnv(t)
+	require.NoError(t, os.Setenv(envPeerIDKey, invalidUUID))
+	require.Error(t, InitUUID())
 
-	err := cfg.reload()
-	assert.Nil(t, err)
+	resetEnv(t)
+	require.NoError(t, os.Setenv(envClusterIDKey, invalidUUID))
+	require.Error(t, InitUUID())
 }