Epoch Commitments

go.mod

@@ -4,6 +4,8 @@ go 1.18
 
 replace github.com/linxGnu/grocksdb => github.com/gohornet/grocksdb v1.6.34-0.20210518222204-d6ea5eedcfb9
 
+replace github.com/lazyledger/smt => github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884
+
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.4
 	github.com/ReneKroon/ttlcache/v2 v2.11.0
@@ -18,6 +20,7 @@ require (
 	github.com/iotaledger/hive.go v0.0.0-20220323102937-0cf57aabb23a
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0
+	github.com/lazyledger/smt v0.2.0
 	github.com/libp2p/go-libp2p v0.15.0
 	github.com/libp2p/go-libp2p-core v0.9.0
 	github.com/libp2p/go-yamux/v2 v2.2.0

go.sum

@@ -144,6 +144,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7
 github.com/capossele/asset-registry v0.0.0-20210521112927-c9d6e74574e8 h1:lCaU8zjlV8f06WK3jlruYmTUPVumZ4sxj0CEyg/k0IU=
 github.com/capossele/asset-registry v0.0.0-20210521112927-c9d6e74574e8/go.mod h1:BXwVCA0+rgYcMKC3vVkfjF+2nXYIYq3h/HndbaCuw08=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
+github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 h1:iRNKw2WmAbVgGMNYzDH5Y2yY3+jyxwEK9Hc5pwIjZAE=
+github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884/go.mod h1:/sdYDakowo/XaxS2Fl7CBqtuf/O2uTqF2zmAUFAtAiw=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -681,6 +683,8 @@ github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8c
 github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
+github.com/lazyledger/smt v0.2.0 h1:0Y64tz0kP5MJrpaO5nTYDLtBidkh6LDi8snGozlYwFU=
+github.com/lazyledger/smt v0.2.0/go.mod h1:9+Pb2/tg1PvEgW7aFx4bFhDE4bvbI03zuJ8kb7nJ9Jc=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=

packages/consensus/finality/finality_gadget.go

@@ -130,6 +130,7 @@ func NewSimpleFinalityGadget(t *tangle.Tangle, opts ...Option) *SimpleFinalityGa
 			MessageConfirmed:      events.NewEvent(tangle.MessageIDCaller),
 			TransactionConfirmed:  events.NewEvent(ledgerstate.TransactionIDEventHandler),
 			BranchConfirmed:       events.NewEvent(ledgerstate.BranchIDEventHandler),
+			BranchRejected:        events.NewEvent(ledgerstate.BranchIDEventHandler),
 			TransactionGoFChanged: events.NewEvent(ledgerstate.TransactionIDEventHandler),
 			BranchGoFChanged:      events.NewEvent(BranchIDGoFEventHandler),
 		},

packages/ledgerstate/branch_dag.go

@@ -164,7 +164,7 @@ func (b *BranchDAG) ResolvePendingBranchIDs(branchIDs BranchIDs) (pendingBranchI
 }
 
 // SetBranchConfirmed sets the InclusionState of the given Branch to be Confirmed.
-func (b *BranchDAG) SetBranchConfirmed(branchID BranchID) (modified bool) {
+func (b *BranchDAG) SetBranchConfirmed(branchID BranchID) (modified bool, rejectedBranches BranchIDs) {
 	b.inclusionStateMutex.Lock()
 	defer b.inclusionStateMutex.Unlock()
 
@@ -193,19 +193,20 @@ func (b *BranchDAG) SetBranchConfirmed(branchID BranchID) (modified bool) {
 		})
 	}
 
+	rejectedBranches = NewBranchIDs()
 	for rejectedWalker.HasNext() {
 		b.Branch(rejectedWalker.Next()).Consume(func(branch *Branch) {
 			if modified = branch.setInclusionState(Rejected); !modified {
 				return
 			}
-
+			rejectedBranches.Add(branch.ID())
 			b.ChildBranches(branch.ID()).Consume(func(childBranch *ChildBranch) {
 				rejectedWalker.Push(childBranch.ChildBranchID())
 			})
 		})
 	}
 
-	return modified
+	return modified, rejectedBranches
 }
 
 // InclusionState returns the InclusionState of the given BranchIDs.

packages/ledgerstate/branch_dag_test.go

@@ -117,7 +117,8 @@ func TestBranchDAG_SetBranchConfirmed(t *testing.T) {
 	branchIDs["Branch7"] = createBranch(t, ledgerstate, "Branch7", NewBranchIDs(MasterBranchID), NewConflictIDs(ConflictID{2}))
 	branchIDs["Branch8"] = createBranch(t, ledgerstate, "Branch8", NewBranchIDs(MasterBranchID), NewConflictIDs(ConflictID{2}))
 
-	assert.True(t, ledgerstate.BranchDAG.SetBranchConfirmed(branchIDs["Branch4"]))
+	modified, _ := ledgerstate.BranchDAG.SetBranchConfirmed(branchIDs["Branch4"])
+	assert.True(t, modified)
 
 	assertInclusionStates(t, ledgerstate, branchIDs, map[string]InclusionState{
 		"Branch2":         Confirmed,
@@ -132,7 +133,8 @@ func TestBranchDAG_SetBranchConfirmed(t *testing.T) {
 		"Branch5+Branch8": Rejected,
 	})
 
-	assert.True(t, ledgerstate.BranchDAG.SetBranchConfirmed(branchIDs["Branch8"]))
+	modified, _ = ledgerstate.BranchDAG.SetBranchConfirmed(branchIDs["Branch8"])
+	assert.True(t, modified)
 
 	// Create a new Branch in an already-decided Conflict set results in straight Reject
 	branchIDs["Branch9"] = createBranch(t, ledgerstate, "Branch9", NewBranchIDs(MasterBranchID), NewConflictIDs(ConflictID{2}))

packages/ledgerstate/ledgerstate_test.go

@@ -26,7 +26,8 @@ func TestLedgerstate_SetBranchConfirmed(t *testing.T) {
 
 	// Mark A as Confirmed
 	{
-		require.True(t, ledgerstate.BranchDAG.SetBranchConfirmed(getSingleBranch(branches, "A")))
+		modified, _ := ledgerstate.BranchDAG.SetBranchConfirmed(getSingleBranch(branches, "A"))
+		require.True(t, modified)
 
 		assertBranchIDs(t, ledgerstate, transactions["A"], branches["A"])
 		assertBranchIDs(t, ledgerstate, transactions["B"], branches["B"])
@@ -93,7 +94,8 @@ func TestLedgerstate_SetBranchConfirmed(t *testing.T) {
 		})
 	}
 
-	require.True(t, ledgerstate.BranchDAG.SetBranchConfirmed(getSingleBranch(branches, "D")))
+	modified, _ := ledgerstate.BranchDAG.SetBranchConfirmed(getSingleBranch(branches, "D"))
+	require.True(t, modified)
 
 	setupScenarioTopTopLayer(t, wallets, outputs, ledgerstate, inputs, manaPledgeID, transactions, branches)
 
@@ -118,8 +120,8 @@ func TestLedgerstate_SetBranchConfirmed(t *testing.T) {
 		assert.Equal(t, Pending, ledgerstate.BranchDAG.InclusionState(branches["H"]))
 		assert.Equal(t, Pending, ledgerstate.BranchDAG.InclusionState(branches["I"]))
 	}
-
-	require.True(t, ledgerstate.BranchDAG.SetBranchConfirmed(getSingleBranch(branches, "H")))
+	modified, _ = ledgerstate.BranchDAG.SetBranchConfirmed(getSingleBranch(branches, "H"))
+	require.True(t, modified)
 
 	setupScenarioTopTopTopLayer(t, wallets, outputs, ledgerstate, inputs, manaPledgeID, transactions)
 

packages/notarization/commitments.go

@@ -0,0 +1,222 @@
+package notarization
+
+import (
+	"sync"
+
+	"github.com/lazyledger/smt"
+	"golang.org/x/crypto/blake2b"
+
+	"github.com/iotaledger/goshimmer/packages/database"
+	"github.com/iotaledger/goshimmer/packages/ledgerstate"
+	"github.com/iotaledger/goshimmer/packages/tangle"
+	"github.com/iotaledger/hive.go/marshalutil"
+)
+
+var (
+	prevTangleRootKey        = []byte("previousTangleRoot")
+	prevStateMutationRootKey = []byte("prevStateMutationRootKey")
+	prevECR                  = []byte("previousEpochCommitmentRoot")
+)
+
+// EpochCommitment contains the ECR and prevECR of an epoch.
+type EpochCommitment struct {
+	EI EI
+	EC []byte
+}
+
+// Commitment is a compressed form of all the information (messages and confirmed value payloads) of an epoch.
+type Commitment struct {
+	EI                EI
+	tangleRoot        *smt.SparseMerkleTree
+	stateMutationRoot *smt.SparseMerkleTree
+	stateRoot         *smt.SparseMerkleTree
+	prevECR           []byte
+}
+
+// NewCommitment returns an empty commitment for the epoch.
+func NewCommitment(ei EI, prevECR, prevMessageRoot, prevTransactionRoot []byte) *Commitment {
+	db, _ := database.NewMemDB()
+	messageIDStore := db.NewStore()
+	messageValueStore := db.NewStore()
+	stateIDStore := db.NewStore()
+	stateValueStore := db.NewStore()
+	stateMutationIDStore := db.NewStore()
+	stateMutationValueStore := db.NewStore()
+
+	hasher, _ := blake2b.New256(nil)
+	commitment := &Commitment{
+		EI:                ei,
+		tangleRoot:        smt.NewSparseMerkleTree(messageIDStore, messageValueStore, hasher),
+		stateMutationRoot: smt.NewSparseMerkleTree(stateIDStore, stateValueStore, hasher),
+		stateRoot:         smt.NewSparseMerkleTree(stateMutationIDStore, stateMutationValueStore, hasher),
+		prevECR:           prevECR,
+	}
+
+	commitment.tangleRoot.Update(prevTangleRootKey, prevMessageRoot)
+	commitment.stateMutationRoot.Update(prevStateMutationRootKey, prevTransactionRoot)
+	return commitment
+}
+
+// TangleRoot returns the root of the tangle sparse merkle tree.
+func (e *Commitment) TangleRoot() []byte {
+	return e.tangleRoot.Root()
+}
+
+// StateMutationRoot returns the root of the state mutation sparse merkle tree.
+func (e *Commitment) StateMutationRoot() []byte {
+	return e.stateMutationRoot.Root()
+}
+
+// StateRoot returns the root of the state sparse merkle tree.
+func (e *Commitment) StateRoot() []byte {
+	return e.stateRoot.Root()
+}
+
+// ECR generates the epoch commitment root.
+func (e *Commitment) ECR() []byte {
+	branch1 := blake2b.Sum256(append(e.prevECR, e.TangleRoot()...))
+	branch2 := blake2b.Sum256(append(e.StateRoot(), e.StateMutationRoot()...))
+	var root []byte
+	root = append(root, branch1[:]...)
+	root = append(root, branch2[:]...)
+	ecr := blake2b.Sum256(root)
+
+	return ecr[:]
+}
+
+// EpochCommitmentFactory manages epoch commitments.
+type EpochCommitmentFactory struct {
+	commitments      map[EI]*Commitment
+	commitmentsMutex sync.RWMutex
+}
+
+// NewEpochCommitmentFactory returns a new commitment factory.
+func NewEpochCommitmentFactory() *EpochCommitmentFactory {
+	return &EpochCommitmentFactory{
+		commitments: make(map[EI]*Commitment),
+	}
+}
+
+// InsertTangleLeaf inserts msg to the Tangle sparse merkle tree.
+func (f *EpochCommitmentFactory) InsertTangleLeaf(ei EI, msgID tangle.MessageID) {
+	commitment := f.getOrCreateCommitment(ei)
+	commitment.tangleRoot.Update(msgID.Bytes(), msgID.Bytes())
+	f.onTangleRootChanged(commitment)
+}
+
+// InsertStateLeaf inserts the outputID to the state sparse merkle tree.
+func (f *EpochCommitmentFactory) InsertStateLeaf(ei EI, outputID ledgerstate.OutputID) {
+	commitment := f.getOrCreateCommitment(ei)
+	commitment.stateRoot.Update(outputID.Bytes(), outputID.Bytes())
+	f.onStateRootChanged(commitment)
+}
+
+// InsertStateMutationLeaf inserts the transaction ID to the state mutation sparse merkle tree.
+func (f *EpochCommitmentFactory) InsertStateMutationLeaf(ei EI, txID ledgerstate.TransactionID) {
+	commitment := f.getOrCreateCommitment(ei)
+	commitment.stateMutationRoot.Update(txID.Bytes(), txID.Bytes())
+	f.onStateMutationRootChanged(commitment)
+}
+
+// RemoveTangleLeaf removes the message ID from the Tangle sparse merkle tree.
+func (f *EpochCommitmentFactory) RemoveTangleLeaf(ei EI, msgID tangle.MessageID) {
+	commitment := f.getOrCreateCommitment(ei)
+	exists, _ := commitment.stateRoot.Has(msgID.Bytes())
+	if exists {
+		commitment.tangleRoot.Delete(msgID.Bytes())
+		f.onTangleRootChanged(commitment)
+	}
+}
+
+// RemoveStateLeaf removes the output ID from the ledgerstate sparse merkle tree.
+func (f *EpochCommitmentFactory) RemoveStateLeaf(ei EI, outID ledgerstate.OutputID) {
+	commitment := f.getOrCreateCommitment(ei)
+	exists, _ := commitment.stateRoot.Has(outID.Bytes())
+	if exists {
+		commitment.stateRoot.Delete(outID.Bytes())
+		f.onStateRootChanged(commitment)
+	}
+}
+
+// GetCommitment returns the commitment with the given ei.
+func (f *EpochCommitmentFactory) GetCommitment(ei EI) *Commitment {
+	f.commitmentsMutex.RLock()
+	defer f.commitmentsMutex.RUnlock()
+	return f.commitments[ei]
+}
+
+// GetEpochCommitment returns the epoch commitment with the given ei.
+func (f *EpochCommitmentFactory) GetEpochCommitment(ei EI) *EpochCommitment {
+	f.commitmentsMutex.RLock()
+	defer f.commitmentsMutex.RUnlock()
+	if commitment, ok := f.commitments[ei]; ok {
+		marshalUtil := marshalutil.New()
+
+		marshalUtil.WriteBytes(commitment.prevECR)
+		marshalUtil.WriteBytes(commitment.ECR())
+		marshalUtil.WriteInt64(int64(ei))
+		ec := blake2b.Sum256(marshalUtil.Bytes())
+
+		return &EpochCommitment{
+			EI: ei,
+			EC: ec[:],
+		}
+	}
+	return nil
+}
+
+func (f *EpochCommitmentFactory) getOrCreateCommitment(ei EI) *Commitment {
+	f.commitmentsMutex.RLock()
+	commitment, ok := f.commitments[ei]
+	f.commitmentsMutex.RUnlock()
+	if !ok {
+		var previousMessageRoot []byte
+		var previousTransactionRoot, previousECR []byte
+
+		if ei > 0 {
+			if previousCommitment := f.GetCommitment(ei - 1); previousCommitment != nil {
+				previousMessageRoot = previousCommitment.TangleRoot()
+				previousTransactionRoot = previousCommitment.StateMutationRoot()
+				previousECR = previousCommitment.ECR()
+			}
+		}
+		commitment = NewCommitment(ei, previousECR, previousMessageRoot, previousTransactionRoot)
+		f.commitmentsMutex.Lock()
+		f.commitments[ei] = commitment
+		f.commitmentsMutex.Unlock()
+	}
+	return commitment
+}
+
+func (f *EpochCommitmentFactory) onTangleRootChanged(commitment *Commitment) {
+	f.commitmentsMutex.RLock()
+	defer f.commitmentsMutex.RUnlock()
+	forwardCommitment, ok := f.commitments[commitment.EI+1]
+	if !ok {
+		return
+	}
+	forwardCommitment.tangleRoot.Update(prevTangleRootKey, commitment.TangleRoot())
+	forwardCommitment.prevECR = commitment.ECR()
+}
+
+func (f *EpochCommitmentFactory) onStateMutationRootChanged(commitment *Commitment) {
+	f.commitmentsMutex.RLock()
+	defer f.commitmentsMutex.RUnlock()
+	forwardCommitment, ok := f.commitments[commitment.EI+1]
+	if !ok {
+		return
+	}
+	forwardCommitment.stateMutationRoot.Update(prevStateMutationRootKey, commitment.StateMutationRoot())
+	forwardCommitment.prevECR = commitment.ECR()
+}
+
+func (f *EpochCommitmentFactory) onStateRootChanged(commitment *Commitment) {
+	f.commitmentsMutex.RLock()
+	defer f.commitmentsMutex.RUnlock()
+	forwardCommitment, ok := f.commitments[commitment.EI+1]
+	if !ok {
+		return
+	}
+	forwardCommitment.stateRoot.Update(prevStateMutationRootKey, commitment.StateMutationRoot())
+	forwardCommitment.prevECR = commitment.ECR()
+}