Streamable Snapshot (#2358)

packages/core/epoch/types.go

@@ -167,3 +167,15 @@ func (e *ECRecord) SetPrevEC(prevEC EC) {
 	e.M.PrevEC = NewMerkleRoot(prevEC[:])
 	e.SetModified()
 }
+
+func (e *ECRecord) Bytes() (bytes []byte, err error) {
+	bytes, err = e.Storable.Bytes()
+	return
+}
+
+func (e *ECRecord) FromBytes(bytes []byte) (err error) {
+	err = e.Storable.FromBytes(bytes)
+	e.SetID(e.EI())
+
+	return
+}

packages/core/ledger/ledger.go

@@ -5,12 +5,12 @@ import (
 	"time"
 
 	"github.com/iotaledger/hive.go/generics/event"
-	"github.com/iotaledger/hive.go/generics/objectstorage"
 	"github.com/iotaledger/hive.go/generics/walker"
 	"github.com/iotaledger/hive.go/syncutils"
 	"github.com/iotaledger/hive.go/types/confirmation"
 
 	"github.com/iotaledger/goshimmer/packages/core/conflictdag"
+	"github.com/iotaledger/goshimmer/packages/core/epoch"
 	"github.com/iotaledger/goshimmer/packages/core/ledger/utxo"
 )
 
@@ -85,9 +85,9 @@ func New(options ...Option) (ledger *Ledger) {
 	return ledger
 }
 
-// LoadSnapshot loads a snapshot of the Ledger from the given snapshot.
-func (l *Ledger) LoadSnapshot(snapshot *Snapshot) {
-	for _, outputWithMetadata := range snapshot.OutputsWithMetadata {
+// LoadOutputWithMetadatas loads OutputWithMetadatas from a snapshot file to the storage.
+func (l *Ledger) LoadOutputWithMetadatas(outputsWithMetadatas []*OutputWithMetadata) {
+	for _, outputWithMetadata := range outputsWithMetadatas {
 		newOutputMetadata := NewOutputMetadata(outputWithMetadata.ID())
 		newOutputMetadata.SetAccessManaPledgeID(outputWithMetadata.AccessManaPledgeID())
 		newOutputMetadata.SetConsensusManaPledgeID(outputWithMetadata.ConsensusManaPledgeID())
@@ -96,9 +96,12 @@ func (l *Ledger) LoadSnapshot(snapshot *Snapshot) {
 		l.Storage.outputStorage.Store(outputWithMetadata.Output()).Release()
 		l.Storage.outputMetadataStorage.Store(newOutputMetadata).Release()
 	}
+}
 
-	for ei := snapshot.FullEpochIndex + 1; ei <= snapshot.DiffEpochIndex; ei++ {
-		epochdiff, exists := snapshot.EpochDiffs[ei]
+// LoadEpochDiffs loads EpochDiffs from a snapshot file to the storage.
+func (l *Ledger) LoadEpochDiffs(header *SnapshotHeader, epochDiffs map[epoch.Index]*EpochDiff) error {
+	for ei := header.FullEpochIndex + 1; ei <= header.DiffEpochIndex; ei++ {
+		epochdiff, exists := epochDiffs[ei]
 		if !exists {
 			panic("epoch diff not found for epoch")
 		}
@@ -118,27 +121,8 @@ func (l *Ledger) LoadSnapshot(snapshot *Snapshot) {
 			l.Storage.outputMetadataStorage.Store(outputMetadata).Release()
 		}
 	}
-}
-
-// TakeSnapshot returns a snapshot of the Ledger state.
-func (l *Ledger) TakeSnapshot() (snapshot *Snapshot) {
-	snapshot = NewSnapshot([]*OutputWithMetadata{})
-	l.Storage.outputMetadataStorage.ForEach(func(key []byte, cachedOutputMetadata *objectstorage.CachedObject[*OutputMetadata]) bool {
-		cachedOutputMetadata.Consume(func(outputMetadata *OutputMetadata) {
-			if outputMetadata.IsSpent() || !l.Utils.OutputConfirmationState(outputMetadata.ID()).IsAccepted() {
-				return
-			}
-
-			l.Storage.CachedOutput(outputMetadata.ID()).Consume(func(output utxo.Output) {
-				outputWithMetadata := NewOutputWithMetadata(output.ID(), output, outputMetadata.CreationTime(), outputMetadata.ConsensusManaPledgeID(), outputMetadata.AccessManaPledgeID())
-				snapshot.OutputsWithMetadata = append(snapshot.OutputsWithMetadata, outputWithMetadata)
-			})
-		})
-
-		return true
-	})
 
-	return snapshot
+	return nil
 }
 
 // SetTransactionInclusionTime sets the inclusion timestamp of a Transaction.

packages/core/ledger/models.go

@@ -10,8 +10,8 @@ import (
 	"github.com/iotaledger/hive.go/stringify"
 	"github.com/iotaledger/hive.go/types/confirmation"
 
-	"github.com/iotaledger/goshimmer/packages/node/clock"
 	"github.com/iotaledger/goshimmer/packages/core/ledger/utxo"
+	"github.com/iotaledger/goshimmer/packages/node/clock"
 )
 
 // region TransactionMetadata //////////////////////////////////////////////////////////////////////////////////////////
@@ -575,7 +575,7 @@ func (e *EpochDiff) Spent() []*OutputWithMetadata {
 
 // Created returns the outputs created for this epoch diff.
 func (e *EpochDiff) Created() []*OutputWithMetadata {
-	return e.M.Spent
+	return e.M.Created
 }
 
 // endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////

packages/core/ledger/snapshot.go

@@ -8,27 +8,42 @@ import (
 
 // Snapshot represents a snapshot of the current ledger state.
 type Snapshot struct {
-	OutputsWithMetadata []*OutputWithMetadata      `serix:"0,lengthPrefixType=uint32"`
-	FullEpochIndex      epoch.Index                `serix:"1"`
-	DiffEpochIndex      epoch.Index                `serix:"2"`
-	EpochDiffs          map[epoch.Index]*EpochDiff `serix:"3,lengthPrefixType=uint32"`
-	LatestECRecord      *epoch.ECRecord            `serix:"4"`
+	Header              *SnapshotHeader            `serix:"0"`
+	OutputsWithMetadata []*OutputWithMetadata      `serix:"1,lengthPrefixType=uint32"`
+	EpochDiffs          map[epoch.Index]*EpochDiff `serix:"2,lengthPrefixType=uint32"`
+}
+
+// SnapshotHeader represents the info of a snapshot.
+type SnapshotHeader struct {
+	OutputWithMetadataCount uint64          `serix:"0"`
+	FullEpochIndex          epoch.Index     `serix:"1"`
+	DiffEpochIndex          epoch.Index     `serix:"2"`
+	LatestECRecord          *epoch.ECRecord `serix:"3"`
 }
 
 // NewSnapshot creates a new Snapshot from the given details.
 func NewSnapshot(outputsWithMetadata []*OutputWithMetadata) (new *Snapshot) {
 	return &Snapshot{
+		Header:              &SnapshotHeader{OutputWithMetadataCount: uint64(len(outputsWithMetadata))},
 		OutputsWithMetadata: outputsWithMetadata,
 	}
 }
 
 // String returns a human-readable version of the Snapshot.
 func (s *Snapshot) String() (humanReadable string) {
-	return stringify.Struct("Snapshot",
-		stringify.StructField("OutputsWithMetadata", s.OutputsWithMetadata),
-		stringify.StructField("FullEpochIndex", s.FullEpochIndex),
-		stringify.StructField("DiffEpochIndex", s.DiffEpochIndex),
-		stringify.StructField("EpochDiffs", s.EpochDiffs),
-		stringify.StructField("LatestECRecord", s.LatestECRecord),
+	structBuilder := stringify.StructBuilder("Snapshot")
+	structBuilder.AddField(stringify.StructField("SnapshotHeader", s.Header))
+	structBuilder.AddField(stringify.StructField("OutputsWithMetadata", s.OutputsWithMetadata))
+	structBuilder.AddField(stringify.StructField("EpochDiffs", s.EpochDiffs))
+	return structBuilder.String()
+}
+
+// String returns a human-readable version of the Snapshot.
+func (h *SnapshotHeader) String() (humanReadable string) {
+	return stringify.Struct("SnapshotHeader",
+		stringify.StructField("OutputWithMetadataCount", h.OutputWithMetadataCount),
+		stringify.StructField("FullEpochIndex", h.FullEpochIndex),
+		stringify.StructField("DiffEpochIndex", h.DiffEpochIndex),
+		stringify.StructField("LatestECRecord", h.LatestECRecord),
 	)
 }

packages/core/notarization/commitments.go

@@ -320,6 +320,17 @@ func (f *EpochCommitmentFactory) loadDiffUTXOs(ei epoch.Index) (spent, created [
 	return
 }
 
+func (f *EpochCommitmentFactory) loadLedgerState(consumer func(*ledger.OutputWithMetadata)) {
+	f.storage.ledgerstateStorage.ForEach(func(_ []byte, cachedOutputWithMetadata *objectstorage.CachedObject[*ledger.OutputWithMetadata]) bool {
+		cachedOutputWithMetadata.Consume(func(outputWithMetadata *ledger.OutputWithMetadata) {
+			consumer(outputWithMetadata)
+		})
+		return true
+	})
+
+	return
+}
+
 // NewCommitment returns an empty commitment for the epoch.
 func (f *EpochCommitmentFactory) newCommitmentTrees(ei epoch.Index) *CommitmentTrees {
 	// Volatile storage for small trees