[Dynamic Protocol State] Protocol state storage

cmd/util/cmd/execution-state-extract/export_report.json

@@ -1,6 +1,6 @@
 {
  "EpochCounter": 0,
- "PreviousStateCommitment": "1c9f9d343cb8d4610e0b2c1eb74d6ea2f2f8aef2d666281dc22870e3efaa607b",
- "CurrentStateCommitment": "1c9f9d343cb8d4610e0b2c1eb74d6ea2f2f8aef2d666281dc22870e3efaa607b",
+ "PreviousStateCommitment": "f057d2597eb52d2c925e182f9b35446acabe74de5797b097cc01dfd2b6585d58",
+ "CurrentStateCommitment": "f057d2597eb52d2c925e182f9b35446acabe74de5797b097cc01dfd2b6585d58",
  "ReportSucceeded": true
 }

consensus/hotstuff/votecollector/staking_vote_processor_test.go

@@ -268,9 +268,7 @@ func TestStakingVoteProcessorV2_BuildVerifyQC(t *testing.T) {
 	}).Sort(order.Canonical)
 
 	leader := stakingSigners[0]
-
-	block := helper.MakeBlock(helper.WithBlockView(view),
-		helper.WithBlockProposer(leader.NodeID))
+	block := helper.MakeBlock(helper.WithBlockView(view), helper.WithBlockProposer(leader.NodeID))
 
 	committee := &mockhotstuff.DynamicCommittee{}
 	committee.On("IdentitiesByEpoch", block.View).Return(stakingSigners.ToSkeleton(), nil)

model/flow/epoch.go

@@ -434,6 +434,11 @@ type EventIDs struct {
 	CommitID Identifier
 }
 
+// ID returns hash of the event IDs.
+func (e *EventIDs) ID() Identifier {
+	return MakeID(e)
+}
+
 func NewEpochStatus(previousSetup, previousCommit, currentSetup, currentCommit, nextSetup, nextCommit Identifier) (*EpochStatus, error) {
 	status := &EpochStatus{
 		PreviousEpoch: EventIDs{

model/flow/identity.go

@@ -570,7 +570,8 @@ func (il IdentityList) SamplePct(pct float64) (IdentityList, error) {
 // Union returns a new identity list containing every identity that occurs in
 // either `il`, or `other`, or both. There are no duplicates in the output,
 // where duplicates are identities with the same node ID.
-// The returned IdentityList is sorted
+// Receiver `il` and/or method input `other` can be nil or empty.
+// The returned IdentityList is sorted in canonical order.
 func (il IdentityList) Union(other IdentityList) IdentityList {
 	maxLen := len(il) + len(other)
 

model/flow/protocol_state.go

@@ -0,0 +1,238 @@
+package flow
+
+import "fmt"
+
+// DynamicIdentityEntry encapsulates nodeID and dynamic portion of identity.
+type DynamicIdentityEntry struct {
+	NodeID  Identifier
+	Dynamic DynamicIdentity
+}
+
+type DynamicIdentityEntryList []*DynamicIdentityEntry
+
+// ProtocolStateEntry represents a snapshot of the identity table (i.e. the set of all notes authorized to
+// be part of the network) at some point in time. It allows to reconstruct the state of identity table using
+// epoch setup events and dynamic identities. It tracks attempts of invalid state transitions.
+// It also holds information about the next epoch, if it has been already committed.
+// This structure is used to persist protocol state in the database.
+//
+// Note that the current implementation does not store the identity table directly. Instead, we store
+// the original events that constituted the _initial_ identity table at the beginning of the epoch
+// plus some modifiers. We intend to restructure this code soon.
+// TODO: https://github.com/onflow/flow-go/issues/4649
+type ProtocolStateEntry struct {
+	// setup and commit event IDs for current epoch.
+	CurrentEpochEventIDs EventIDs
+	// setup and commit event IDs for previous epoch.
+	PreviousEpochEventIDs EventIDs
+	// Part of identity table that can be changed during the epoch.
+	// Always sorted in canonical order.
+	Identities DynamicIdentityEntryList
+	// InvalidStateTransitionAttempted encodes whether an invalid state transition
+	// has been detected in this fork. When this happens, epoch fallback is triggered
+	// AFTER the fork is finalized.
+	InvalidStateTransitionAttempted bool
+	// NextEpochProtocolState describes protocol state of the next epoch
+	NextEpochProtocolState *ProtocolStateEntry
+}
+
+// RichProtocolStateEntry is a ProtocolStateEntry which has additional fields that are cached
+// from storage layer for convenience.
+// Using this structure instead of ProtocolStateEntry allows us to avoid querying
+// the database for epoch setups and commits and full identity table.
+// It holds several invariants, such as:
+//   - CurrentEpochSetup and CurrentEpochCommit are for the same epoch. Never nil.
+//   - PreviousEpochSetup and PreviousEpochCommit are for the same epoch. Can be nil.
+//   - Identities is a full identity table for the current epoch.
+//     Identities are sorted in canonical order. Without duplicates. Never nil.
+//   - NextEpochProtocolState is a protocol state for the next epoch. Can be nil.
+type RichProtocolStateEntry struct {
+	*ProtocolStateEntry
+
+	CurrentEpochSetup   *EpochSetup
+	CurrentEpochCommit  *EpochCommit
+	PreviousEpochSetup  *EpochSetup
+	PreviousEpochCommit *EpochCommit
+	Identities          IdentityList
+
+	NextEpochProtocolState *RichProtocolStateEntry
+}
+
+// NewRichProtocolStateEntry constructs a rich protocol state entry from a protocol state entry and additional data.
+// No errors are expected during normal operation.
+func NewRichProtocolStateEntry(
+	protocolState *ProtocolStateEntry,
+	previousEpochSetup *EpochSetup,
+	previousEpochCommit *EpochCommit,
+	currentEpochSetup *EpochSetup,
+	currentEpochCommit *EpochCommit,
+	nextEpochSetup *EpochSetup,
+	nextEpochCommit *EpochCommit,
+) (*RichProtocolStateEntry, error) {
+	result := &RichProtocolStateEntry{
+		ProtocolStateEntry:     protocolState,
+		CurrentEpochSetup:      currentEpochSetup,
+		CurrentEpochCommit:     currentEpochCommit,
+		PreviousEpochSetup:     previousEpochSetup,
+		PreviousEpochCommit:    previousEpochCommit,
+		Identities:             nil,
+		NextEpochProtocolState: nil,
+	}
+
+	var err error
+	// if next epoch has been already committed, fill in data for it as well.
+	if protocolState.NextEpochProtocolState != nil {
+		// sanity check consistency of input data
+		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID != nextEpochSetup.ID() {
+			return nil, fmt.Errorf("inconsistent EpochSetup for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
+				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID, nextEpochSetup.ID())
+		}
+		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID != nextEpochCommit.ID() {
+			return nil, fmt.Errorf("inconsistent EpochCommit for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
+				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID, nextEpochCommit.ID())
+		}
+
+		// sanity check consistency of input data
+		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID != nextEpochSetup.ID() {
+			return nil, fmt.Errorf("inconsistent EpochSetup for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
+				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID, nextEpochSetup.ID())
+		}
+		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID != nextEpochCommit.ID() {
+			return nil, fmt.Errorf("inconsistent EpochCommit for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
+				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID, nextEpochCommit.ID())
+		}
+
+		// if next epoch is available, it means that we have observed epoch setup event and we are not anymore in staking phase,
+		// so we need to build the identity table using current and next epoch setup events.
+		// so we need to build the identity table using current and next epoch setup events.
+		result.Identities, err = buildIdentityTable(
+			protocolState.Identities,
+			currentEpochSetup.Participants,
+			nextEpochSetup.Participants,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("could not build identity table for setup/commit phase: %w", err)
+		}
+
+		nextEpochProtocolState := protocolState.NextEpochProtocolState
+		nextEpochIdentityTable, err := buildIdentityTable(
+			nextEpochProtocolState.Identities,
+			nextEpochSetup.Participants,
+			currentEpochSetup.Participants,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("could not build next epoch identity table: %w", err)
+		}
+
+		// fill identities for next epoch
+		result.NextEpochProtocolState = &RichProtocolStateEntry{
+			ProtocolStateEntry:     nextEpochProtocolState,
+			CurrentEpochSetup:      nextEpochSetup,
+			CurrentEpochCommit:     nextEpochCommit,
+			PreviousEpochSetup:     result.CurrentEpochSetup,  // previous epoch setup is current epoch setup
+			PreviousEpochCommit:    result.CurrentEpochCommit, // previous epoch setup is current epoch setup
+			Identities:             nextEpochIdentityTable,
+			NextEpochProtocolState: nil, // always nil
+		}
+	} else {
+		// if next epoch is not yet created, it means that we are in staking phase,
+		// so we need to build the identity table using previous and current epoch setup events.
+		var otherIdentities IdentityList
+		if previousEpochSetup != nil {
+			otherIdentities = previousEpochSetup.Participants
+		}
+		result.Identities, err = buildIdentityTable(
+			protocolState.Identities,
+			currentEpochSetup.Participants,
+			otherIdentities,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("could not build identity table for staking phase: %w", err)
+		}
+	}
+
+	return result, nil
+}
+
+// ID returns hash of entry by hashing all fields.
+func (e *ProtocolStateEntry) ID() Identifier {
+	if e == nil {
+		return ZeroID
+	}
+	body := struct {
+		CurrentEpochEventIDs            Identifier
+		PreviousEpochEventIDs           Identifier
+		Identities                      DynamicIdentityEntryList
+		InvalidStateTransitionAttempted bool
+		NextEpochProtocolStateID        Identifier
+	}{
+		CurrentEpochEventIDs:            e.CurrentEpochEventIDs.ID(),
+		PreviousEpochEventIDs:           e.PreviousEpochEventIDs.ID(),
+		Identities:                      e.Identities,
+		InvalidStateTransitionAttempted: e.InvalidStateTransitionAttempted,
+		NextEpochProtocolStateID:        e.NextEpochProtocolState.ID(),
+	}
+	return MakeID(body)
+}
+
+// Sorted returns whether the list is sorted by the input ordering.
+func (ll DynamicIdentityEntryList) Sorted(less IdentifierOrder) bool {
+	for i := 0; i < len(ll)-1; i++ {
+		a := ll[i]
+		b := ll[i+1]
+		if !less(a.NodeID, b.NodeID) {
+			return false
+		}
+	}
+	return true
+}
+
+// buildIdentityTable constructs the full identity table for the target epoch by combining data from:
+//  1. The target epoch's Dynamic Identities.
+//  2. The target epoch's IdentitySkeletons
+//     (recorded in EpochSetup event and immutable throughout the epoch).
+//  3. [optional] An adjacent epoch's IdentitySkeletons (can be empty or nil), as recorded in the
+//     adjacent epoch's setup event. For a target epoch N, the epochs N-1 and N+1 are defined to be
+//     adjacent. Adjacent epochs do not _necessarily_ exist (e.g. consider a spork comprising only
+//     a single epoch), in which case this input is nil or empty.
+//
+// It also performs sanity checks to make sure that the data is consistent.
+// No errors are expected during normal operation.
+func buildIdentityTable(
+	targetEpochDynamicIdentities DynamicIdentityEntryList,
+	targetEpochIdentitySkeletons IdentityList, // TODO: change to `IdentitySkeletonList`
+	adjacentEpochIdentitySkeletons IdentityList, // TODO: change to `IdentitySkeletonList`
+) (IdentityList, error) {
+	// produce a unique set for current and previous epoch participants
+	allEpochParticipants := targetEpochIdentitySkeletons.Union(adjacentEpochIdentitySkeletons)
+	// sanity check: size of identities should be equal to previous and current epoch participants combined
+	if len(allEpochParticipants) != len(targetEpochDynamicIdentities) {
+		return nil, fmt.Errorf("invalid number of identities in protocol state: expected %d, got %d", len(allEpochParticipants), len(targetEpochDynamicIdentities))
+	}
+
+	// build full identity table for current epoch
+	var result IdentityList
+	for i, identity := range targetEpochDynamicIdentities {
+		// sanity check: identities should be sorted in canonical order
+		if identity.NodeID != allEpochParticipants[i].NodeID {
+			return nil, fmt.Errorf("identites in protocol state are not in canonical order: expected %s, got %s", allEpochParticipants[i].NodeID, identity.NodeID)
+		}
+		result = append(result, &Identity{
+			IdentitySkeleton: allEpochParticipants[i].IdentitySkeleton,
+			DynamicIdentity:  identity.Dynamic,
+		})
+	}
+	return result, nil
+}
+
+// DynamicIdentityEntryListFromIdentities converts IdentityList to DynamicIdentityEntryList.
+func DynamicIdentityEntryListFromIdentities(identities IdentityList) DynamicIdentityEntryList {
+	dynamicIdentities := make(DynamicIdentityEntryList, 0, len(identities))
+	for _, identity := range identities {
+		dynamicIdentities = append(dynamicIdentities, &DynamicIdentityEntry{
+			NodeID:  identity.NodeID,
+			Dynamic: identity.DynamicIdentity,
+		})
+	}
+	return dynamicIdentities
+}