Merge pull request #1169 from iotaledger/develop

Merge v0.5.5 into master

CHANGELOG.md

@@ -1,3 +1,13 @@
+# v0.5.5 - 2021-04-01
+* Integrate Mana with FPC
+* Integrate Mana with the Autopeering
+* Add several FPC optimizations
+* Add dRNG diagnostic API
+* Simplify memory usage of dashboard and align to Grafana
+* Add a chart for stored, solidifier, scheduler and booker MPS
+* Update to latest hive.go
+* **Breaking**: bumps network and database versions
+
 # v0.5.4 - 2021-03-29
 * Add new diagnostic APIs
 * Add new docs sections

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/golang/protobuf v1.4.3
 	github.com/golang/snappy v0.0.2 // indirect
 	github.com/gorilla/websocket v1.4.2
-	github.com/iotaledger/hive.go v0.0.0-20210317185122-1ac328c4054f
+	github.com/iotaledger/hive.go v0.0.0-20210327095456-8b9171badf36
 	github.com/kr/pretty v0.2.1 // indirect
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0

go.sum

@@ -404,8 +404,8 @@ github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:q
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/iotaledger/hive.go v0.0.0-20210317185122-1ac328c4054f h1:+hbQVRGodH7rBJS/cPG9qgdz37L54WRshFZYf6Cdv2o=
-github.com/iotaledger/hive.go v0.0.0-20210317185122-1ac328c4054f/go.mod h1:K/0FfpFpR4elUAvRFMn8UNfmiQmKD7EcZdWngRLi2hw=
+github.com/iotaledger/hive.go v0.0.0-20210327095456-8b9171badf36 h1:WBxsDCexBJOz3ObOFazSz4uwYmS/zjOLurmWdLGCURk=
+github.com/iotaledger/hive.go v0.0.0-20210327095456-8b9171badf36/go.mod h1:NyBg/Ny7FFAdDs59zdwTVoysU2ZbJVQnRwyLIDFKJYA=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -648,6 +648,7 @@ github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ
 github.com/libp2p/go-yamux v1.3.6/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
+github.com/linxGnu/grocksdb v1.6.32/go.mod h1:/+iSQrn7Izt6kFhHBQvcE6FkklsKXa8hc35pFyFDrDw=
 github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -1379,8 +1380,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

packages/consensus/fcob/consensusmechanism.go

@@ -308,6 +308,13 @@ func (f *ConsensusMechanism) createMessageOpinion(messageID tangle.MessageID, wa
 	// trigger TransactionOpinionFormed if the message contains a transaction
 	f.tangle.Utils.ComputeIfTransaction(messageID, func(transactionID ledgerstate.TransactionID) {
 		if f.tangle.LedgerState.BranchInclusionState(f.tangle.LedgerState.BranchID(transactionID)) == ledgerstate.Confirmed {
+			// skip reattachments
+			for _, attachmentID := range f.tangle.Storage.AttachmentMessageIDs(transactionID) {
+				if f.messageOpinionTriggered(attachmentID) {
+					return
+				}
+			}
+
 			f.tangle.ConsensusManager.Events.TransactionConfirmed.Trigger(messageID)
 		}
 	})

packages/ledgerstate/branch_dag.go

@@ -266,7 +266,7 @@ func (b *BranchDAG) ChildBranches(branchID BranchID) (cachedChildBranches Cached
 		cachedChildBranches = append(cachedChildBranches, &CachedChildBranch{CachedObject: cachedObject})
 
 		return true
-	}, objectstorage.WithPrefix(branchID.Bytes()))
+	}, objectstorage.WithIteratorPrefix(branchID.Bytes()))
 
 	return
 }
@@ -294,7 +294,7 @@ func (b *BranchDAG) ConflictMembers(conflictID ConflictID) (cachedConflictMember
 		cachedConflictMembers = append(cachedConflictMembers, &CachedConflictMember{CachedObject: cachedObject})
 
 		return true
-	}, objectstorage.WithPrefix(conflictID.Bytes()))
+	}, objectstorage.WithIteratorPrefix(conflictID.Bytes()))
 
 	return
 }

packages/ledgerstate/utxo_dag.go

@@ -242,7 +242,7 @@ func (u *UTXODAG) Consumers(outputID OutputID) (cachedConsumers CachedConsumers)
 		cachedConsumers = append(cachedConsumers, &CachedConsumer{CachedObject: cachedObject})
 
 		return true
-	}, objectstorage.WithPrefix(outputID.Bytes()))
+	}, objectstorage.WithIteratorPrefix(outputID.Bytes()))
 
 	return
 }
@@ -299,7 +299,7 @@ func (u *UTXODAG) AddressOutputMapping(address Address) (cachedAddressOutputMapp
 	u.addressOutputMappingStorage.ForEach(func(key []byte, cachedObject objectstorage.CachedObject) bool {
 		cachedAddressOutputMappings = append(cachedAddressOutputMappings, &CachedAddressOutputMapping{cachedObject})
 		return true
-	}, objectstorage.WithPrefix(address.Bytes()))
+	}, objectstorage.WithIteratorPrefix(address.Bytes()))
 	return
 }
 

packages/mana/accessbasevector.go

@@ -148,6 +148,53 @@ func (a *AccessBaseManaVector) GetHighestManaNodes(n uint) (res []Node, t time.T
 	return
 }
 
+// GetHighestManaNodesFraction returns the highest mana that own 'p' percent of total mana.
+// It also updates the mana values for each node.
+// If p is zero or greater than one, it returns all nodes.
+func (a *AccessBaseManaVector) GetHighestManaNodesFraction(p float64) (res []Node, t time.Time, err error) {
+	totalMana := 0.0
+	err = func() error {
+		// don't lock the vector after this func returns
+		a.Lock()
+		defer a.Unlock()
+		t = time.Now()
+		for ID := range a.vector {
+			var mana float64
+			mana, _, err = a.getMana(ID, t)
+			if err != nil {
+				return err
+			}
+			res = append(res, Node{
+				ID:   ID,
+				Mana: mana,
+			})
+			totalMana += mana
+		}
+		return nil
+	}()
+	if err != nil {
+		return nil, t, err
+	}
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Mana > res[j].Mana
+	})
+
+	// how much mana is p percent of total mana
+	manaThreshold := p * totalMana
+	// include nodes as long as their counted mana is less than the threshold
+	manaCounted := 0.0
+	var n uint
+	for n = 0; int(n) < len(res) && manaCounted < manaThreshold; n++ {
+		manaCounted += res[n].Mana
+	}
+
+	if n == 0 || int(n) >= len(res) {
+		return
+	}
+	res = res[:n]
+	return res, t, err
+}
+
 // SetMana sets the base mana for a node.
 func (a *AccessBaseManaVector) SetMana(nodeID identity.ID, bm BaseMana) {
 	a.Lock()

packages/mana/accessbasevector_test.go

@@ -538,6 +538,59 @@ func TestAccessBaseManaVector_GetHighestManaNodes(t *testing.T) {
 	}
 }
 
+func TestAccessBaseManaVector_GetHighestManaNodesFraction(t *testing.T) {
+	bmv, err := NewBaseManaVector(AccessMana)
+	assert.NoError(t, err)
+
+	nodeIDs := make([]identity.ID, 10)
+
+	baseTime = time.Now()
+
+	for i := 0; i < 10; i++ {
+		nodeIDs[i] = randNodeID()
+		bmv.SetMana(nodeIDs[i], &AccessBaseMana{
+			BaseMana2:          float64(i),
+			EffectiveBaseMana2: float64(i),
+			LastUpdated:        baseTime,
+		})
+	}
+
+	// requesting minus value
+	result, _, err := bmv.GetHighestManaNodesFraction(-0.1)
+	assert.NoError(t, err)
+	assert.Equal(t, 10, len(result))
+	assert.Equal(t, nodeIDs[9], result[0].ID)
+	assert.InDelta(t, 9.0, result[0].Mana, delta)
+
+	// requesting the holders of top 5% of mana
+	result, _, err = bmv.GetHighestManaNodesFraction(0.1)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(result))
+	assert.Equal(t, nodeIDs[9], result[0].ID)
+	assert.InDelta(t, 9.0, result[0].Mana, delta)
+
+	// requesting holders of top 50% of mana
+	result, _, err = bmv.GetHighestManaNodesFraction(0.5)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(result))
+	assert.InDelta(t, 9.0, result[0].Mana, delta)
+	for index, value := range result {
+		if index < 2 {
+			// it's greater than the next one
+			assert.True(t, value.Mana > result[index+1].Mana)
+		}
+		assert.Equal(t, nodeIDs[9-index], value.ID)
+	}
+
+	// requesting more, than there currently are in the vector
+	result, _, err = bmv.GetHighestManaNodesFraction(1.1)
+	assert.NoError(t, err)
+	assert.Equal(t, 10, len(result))
+	for index, value := range result {
+		assert.Equal(t, nodeIDs[9-index], value.ID)
+	}
+}
+
 func TestAccessBaseManaVector_SetMana(t *testing.T) {
 	bmv, err := NewBaseManaVector(AccessMana)
 	assert.NoError(t, err)

packages/mana/basevector.go

@@ -27,6 +27,8 @@ type BaseManaVector interface {
 	GetManaMap(...time.Time) (NodeMap, time.Time, error)
 	// GetHighestManaNodes returns the n highest mana nodes in descending order.
 	GetHighestManaNodes(uint) ([]Node, time.Time, error)
+	// GetHighestManaNodesFraction returns the highest mana that own 'p' percent of total mana.
+	GetHighestManaNodesFraction(p float64) ([]Node, time.Time, error)
 	// SetMana sets the base mana for a node.
 	SetMana(identity.ID, BaseMana)
 	// ForEach executes a callback function for each entry in the vector.

packages/mana/consensusbasevector.go

@@ -1,6 +1,7 @@
 package mana
 
 import (
+	"bytes"
 	"fmt"
 	"sort"
 	"sync"
@@ -232,6 +233,59 @@ func (c *ConsensusBaseManaVector) GetHighestManaNodes(n uint) (res []Node, t tim
 	return
 }
 
+// GetHighestManaNodesFraction returns the highest mana that own 'p' percent of total mana.
+// It also updates the mana values for each node.
+// If p is zero or greater than one, it returns all nodes.
+func (c *ConsensusBaseManaVector) GetHighestManaNodesFraction(p float64) (res []Node, t time.Time, err error) {
+	emptyNodeID := identity.ID{}
+	totalMana := 0.0
+	err = func() error {
+		// don't lock the vector after this func returns
+		c.Lock()
+		defer c.Unlock()
+		t = time.Now()
+		for ID := range c.vector {
+			// skip the empty node ID
+			if bytes.Equal(ID[:], emptyNodeID[:]) {
+				continue
+			}
+
+			var mana float64
+			mana, _, err = c.getMana(ID, t)
+			if err != nil {
+				return err
+			}
+			res = append(res, Node{
+				ID:   ID,
+				Mana: mana,
+			})
+			totalMana += mana
+		}
+		return nil
+	}()
+	if err != nil {
+		return nil, t, err
+	}
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Mana > res[j].Mana
+	})
+
+	// how much mana is p percent of total mana
+	manaThreshold := p * totalMana
+	// include nodes as long as their counted mana is less than the threshold
+	manaCounted := 0.0
+	var n uint
+	for n = 0; int(n) < len(res) && manaCounted < manaThreshold; n++ {
+		manaCounted += res[n].Mana
+	}
+
+	if n == 0 || int(n) >= len(res) {
+		return
+	}
+	res = res[:n]
+	return res, t, err
+}
+
 // SetMana sets the base mana for a node.
 func (c *ConsensusBaseManaVector) SetMana(nodeID identity.ID, bm BaseMana) {
 	c.Lock()

packages/mana/consensusbasevector_test.go

@@ -492,6 +492,59 @@ func TestConsensusBaseManaVector_GetHighestManaNodes(t *testing.T) {
 	}
 }
 
+func TestConsensusBaseManaVector_GetHighestManaNodesFraction(t *testing.T) {
+	bmv, err := NewBaseManaVector(ConsensusMana)
+	assert.NoError(t, err)
+
+	nodeIDs := make([]identity.ID, 10)
+
+	baseTime = time.Now()
+
+	for i := 0; i < 10; i++ {
+		nodeIDs[i] = randNodeID()
+		bmv.SetMana(nodeIDs[i], &ConsensusBaseMana{
+			BaseMana1:          float64(i),
+			EffectiveBaseMana1: float64(i),
+			LastUpdated:        baseTime,
+		})
+	}
+
+	// requesting minus value
+	result, _, err := bmv.GetHighestManaNodesFraction(-0.1)
+	assert.NoError(t, err)
+	assert.Equal(t, 10, len(result))
+	assert.Equal(t, nodeIDs[9], result[0].ID)
+	assert.InDelta(t, 9.0, result[0].Mana, delta)
+
+	// requesting the holders of top 10% of mana
+	result, _, err = bmv.GetHighestManaNodesFraction(0.2)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(result))
+	assert.Equal(t, nodeIDs[9], result[0].ID)
+	assert.InDelta(t, 9.0, result[0].Mana, delta)
+
+	// requesting holders of top 50% of mana
+	result, _, err = bmv.GetHighestManaNodesFraction(0.5)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(result))
+	assert.InDelta(t, 9.0, result[0].Mana, delta)
+	for index, value := range result {
+		if index < 2 {
+			// it's greater than the next one
+			assert.True(t, value.Mana > result[index+1].Mana)
+		}
+		assert.Equal(t, nodeIDs[9-index], value.ID)
+	}
+
+	// requesting more, than there currently are in the vector
+	result, _, err = bmv.GetHighestManaNodesFraction(1.1)
+	assert.NoError(t, err)
+	assert.Equal(t, 10, len(result))
+	for index, value := range result {
+		assert.Equal(t, nodeIDs[9-index], value.ID)
+	}
+}
+
 func TestConsensusBaseManaVector_SetMana(t *testing.T) {
 	bmv, err := NewBaseManaVector(ConsensusMana)
 	assert.NoError(t, err)