diff --git a/Makefile b/Makefile index f58f1e928f0..724f84d5ed7 100644 --- a/Makefile +++ b/Makefile @@ -199,6 +199,8 @@ generate-mocks: install-mock-generators mockery --name 'API' --dir="./access" --case=underscore --output="./access/mock" --outpkg="mock" mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock" mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock" + mockery --name 'BlockTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock" --outpkg="mock" + mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock" --outpkg="mock" mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock" mockery --name 'Communicator' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock" diff --git a/access/api.go b/access/api.go index 757ae2c9805..a31f8ca0841 100644 --- a/access/api.go +++ b/access/api.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" ) @@ -52,6 +53,147 @@ type API interface { GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) + + // SubscribeBlocks + + // SubscribeBlocksFromStartBlockID subscribes to the finalized or sealed blocks starting at the requested + // start block id, up until the latest available block. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block as it becomes available. + // + // Each block is filtered by the provided block status, and only + // those blocks that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - startBlockID: The identifier of the starting block. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlocksFromStartBlockID will return a failed subscription. + SubscribeBlocksFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription + // SubscribeBlocksFromStartHeight subscribes to the finalized or sealed blocks starting at the requested + // start block height, up until the latest available block. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block as it becomes available. + // + // Each block is filtered by the provided block status, and only + // those blocks that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - startHeight: The height of the starting block. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlocksFromStartHeight will return a failed subscription. + SubscribeBlocksFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription + // SubscribeBlocksFromLatest subscribes to the finalized or sealed blocks starting at the latest sealed block, + // up until the latest available block. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block as it becomes available. + // + // Each block is filtered by the provided block status, and only + // those blocks that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlocksFromLatest will return a failed subscription. + SubscribeBlocksFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription + + // SubscribeHeaders + + // SubscribeBlockHeadersFromStartBlockID streams finalized or sealed block headers starting at the requested + // start block id, up until the latest available block header. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block header as it becomes available. + // + // Each block header are filtered by the provided block status, and only + // those block headers that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - startBlockID: The identifier of the starting block. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlockHeadersFromStartBlockID will return a failed subscription. + SubscribeBlockHeadersFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription + // SubscribeBlockHeadersFromStartHeight streams finalized or sealed block headers starting at the requested + // start block height, up until the latest available block header. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block header as it becomes available. + // + // Each block header are filtered by the provided block status, and only + // those block headers that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - startHeight: The height of the starting block. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlockHeadersFromStartHeight will return a failed subscription. + SubscribeBlockHeadersFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription + // SubscribeBlockHeadersFromLatest streams finalized or sealed block headers starting at the latest sealed block, + // up until the latest available block header. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block header as it becomes available. + // + // Each block header are filtered by the provided block status, and only + // those block headers that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlockHeadersFromLatest will return a failed subscription. + SubscribeBlockHeadersFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription + + // Subscribe digests + + // SubscribeBlockDigestsFromStartBlockID streams finalized or sealed lightweight block starting at the requested + // start block id, up until the latest available block. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block as it becomes available. + // + // Each lightweight block are filtered by the provided block status, and only + // those blocks that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - startBlockID: The identifier of the starting block. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlockDigestsFromStartBlockID will return a failed subscription. + SubscribeBlockDigestsFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription + // SubscribeBlockDigestsFromStartHeight streams finalized or sealed lightweight block starting at the requested + // start block height, up until the latest available block. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block as it becomes available. + // + // Each lightweight block are filtered by the provided block status, and only + // those blocks that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - startHeight: The height of the starting block. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlockDigestsFromStartHeight will return a failed subscription. + SubscribeBlockDigestsFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription + // SubscribeBlockDigestsFromLatest streams finalized or sealed lightweight block starting at the latest sealed block, + // up until the latest available block. Once the latest is + // reached, the stream will remain open and responses are sent for each new + // block as it becomes available. + // + // Each lightweight block are filtered by the provided block status, and only + // those blocks that match the status are returned. + // + // Parameters: + // - ctx: Context for the operation. + // - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. + // + // If invalid parameters will be supplied SubscribeBlockDigestsFromLatest will return a failed subscription. + SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription } // TODO: Combine this with flow.TransactionResult? diff --git a/access/handler.go b/access/handler.go index 8059cc9bd7b..e7ed10b744e 100644 --- a/access/handler.go +++ b/access/handler.go @@ -5,9 +5,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -17,6 +20,7 @@ import ( ) type Handler struct { + subscription.StreamingData api API chain flow.Chain signerIndicesDecoder hotstuff.BlockSignerDecoder @@ -29,8 +33,28 @@ type HandlerOption func(*Handler) var _ access.AccessAPIServer = (*Handler)(nil) -func NewHandler(api API, chain flow.Chain, finalizedHeader module.FinalizedHeaderCache, me module.Local, options ...HandlerOption) *Handler { +// sendSubscribeBlocksResponseFunc is a callback function used to send +// SubscribeBlocksResponse to the client stream. +type sendSubscribeBlocksResponseFunc func(*access.SubscribeBlocksResponse) error + +// sendSubscribeBlockHeadersResponseFunc is a callback function used to send +// SubscribeBlockHeadersResponse to the client stream. +type sendSubscribeBlockHeadersResponseFunc func(*access.SubscribeBlockHeadersResponse) error + +// sendSubscribeBlockDigestsResponseFunc is a callback function used to send +// SubscribeBlockDigestsResponse to the client stream. +type sendSubscribeBlockDigestsResponseFunc func(*access.SubscribeBlockDigestsResponse) error + +func NewHandler( + api API, + chain flow.Chain, + finalizedHeader module.FinalizedHeaderCache, + me module.Local, + maxStreams uint32, + options ...HandlerOption, +) *Handler { h := &Handler{ + StreamingData: subscription.NewStreamingData(maxStreams), api: api, chain: chain, finalizedHeaderCache: finalizedHeader, @@ -701,6 +725,372 @@ func (h *Handler) GetExecutionResultByID(ctx context.Context, req *access.GetExe }, nil } +// SubscribeBlocksFromStartBlockID handles subscription requests for blocks started from block id. +// It takes a SubscribeBlocksFromStartBlockIDRequest and an AccessAPI_SubscribeBlocksFromStartBlockIDServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if invalid startBlockID provided or unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block to message or could not send response. +func (h *Handler) SubscribeBlocksFromStartBlockID(request *access.SubscribeBlocksFromStartBlockIDRequest, stream access.AccessAPI_SubscribeBlocksFromStartBlockIDServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + startBlockID, blockStatus, err := h.getSubscriptionDataFromStartBlockID(request.GetStartBlockId(), request.GetBlockStatus()) + if err != nil { + return err + } + + sub := h.api.SubscribeBlocksFromStartBlockID(stream.Context(), startBlockID, blockStatus) + return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus)) +} + +// SubscribeBlocksFromStartHeight handles subscription requests for blocks started from block height. +// It takes a SubscribeBlocksFromStartHeightRequest and an AccessAPI_SubscribeBlocksFromStartHeightServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block to message or could not send response. +func (h *Handler) SubscribeBlocksFromStartHeight(request *access.SubscribeBlocksFromStartHeightRequest, stream access.AccessAPI_SubscribeBlocksFromStartHeightServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + blockStatus := convert.MessageToBlockStatus(request.GetBlockStatus()) + err := checkBlockStatus(blockStatus) + if err != nil { + return err + } + + sub := h.api.SubscribeBlocksFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus) + return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus)) +} + +// SubscribeBlocksFromLatest handles subscription requests for blocks started from latest sealed block. +// It takes a SubscribeBlocksFromLatestRequest and an AccessAPI_SubscribeBlocksFromLatestServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block to message or could not send response. +func (h *Handler) SubscribeBlocksFromLatest(request *access.SubscribeBlocksFromLatestRequest, stream access.AccessAPI_SubscribeBlocksFromLatestServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + blockStatus := convert.MessageToBlockStatus(request.GetBlockStatus()) + err := checkBlockStatus(blockStatus) + if err != nil { + return err + } + + sub := h.api.SubscribeBlocksFromLatest(stream.Context(), blockStatus) + return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus)) +} + +// handleBlocksResponse handles the subscription to block updates and sends +// the subscribed block information to the client via the provided stream. +// +// Parameters: +// - send: The function responsible for sending the block response to the client. +// - fullBlockResponse: A boolean indicating whether to include full block responses. +// - blockStatus: The current block status. +// +// Returns a function that can be used as a callback for block updates. +// +// This function is designed to be used as a callback for block updates in a subscription. +// It takes a block, processes it, and sends the corresponding response to the client using the provided send function. +// +// Expected errors during normal operation: +// - codes.Internal: If cannot convert a block to a message or the stream could not send a response. +func (h *Handler) handleBlocksResponse(send sendSubscribeBlocksResponseFunc, fullBlockResponse bool, blockStatus flow.BlockStatus) func(*flow.Block) error { + return func(block *flow.Block) error { + msgBlockResponse, err := h.blockResponse(block, fullBlockResponse, blockStatus) + if err != nil { + return rpc.ConvertError(err, "could not convert block to message", codes.Internal) + } + + err = send(&access.SubscribeBlocksResponse{ + Block: msgBlockResponse.Block, + }) + if err != nil { + return rpc.ConvertError(err, "could not send response", codes.Internal) + } + + return nil + } +} + +// SubscribeBlockHeadersFromStartBlockID handles subscription requests for block headers started from block id. +// It takes a SubscribeBlockHeadersFromStartBlockIDRequest and an AccessAPI_SubscribeBlockHeadersFromStartBlockIDServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block header information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if invalid startBlockID provided or unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block header to message or could not send response. +func (h *Handler) SubscribeBlockHeadersFromStartBlockID(request *access.SubscribeBlockHeadersFromStartBlockIDRequest, stream access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + startBlockID, blockStatus, err := h.getSubscriptionDataFromStartBlockID(request.GetStartBlockId(), request.GetBlockStatus()) + if err != nil { + return err + } + + sub := h.api.SubscribeBlockHeadersFromStartBlockID(stream.Context(), startBlockID, blockStatus) + return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send)) +} + +// SubscribeBlockHeadersFromStartHeight handles subscription requests for block headers started from block height. +// It takes a SubscribeBlockHeadersFromStartHeightRequest and an AccessAPI_SubscribeBlockHeadersFromStartHeightServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block header information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block header to message or could not send response. +func (h *Handler) SubscribeBlockHeadersFromStartHeight(request *access.SubscribeBlockHeadersFromStartHeightRequest, stream access.AccessAPI_SubscribeBlockHeadersFromStartHeightServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + blockStatus := convert.MessageToBlockStatus(request.GetBlockStatus()) + err := checkBlockStatus(blockStatus) + if err != nil { + return err + } + + sub := h.api.SubscribeBlockHeadersFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus) + return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send)) +} + +// SubscribeBlockHeadersFromLatest handles subscription requests for block headers started from latest sealed block. +// It takes a SubscribeBlockHeadersFromLatestRequest and an AccessAPI_SubscribeBlockHeadersFromLatestServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block header information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block header to message or could not send response. +func (h *Handler) SubscribeBlockHeadersFromLatest(request *access.SubscribeBlockHeadersFromLatestRequest, stream access.AccessAPI_SubscribeBlockHeadersFromLatestServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + blockStatus := convert.MessageToBlockStatus(request.GetBlockStatus()) + err := checkBlockStatus(blockStatus) + if err != nil { + return err + } + + sub := h.api.SubscribeBlockHeadersFromLatest(stream.Context(), blockStatus) + return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send)) +} + +// handleBlockHeadersResponse handles the subscription to block updates and sends +// the subscribed block header information to the client via the provided stream. +// +// Parameters: +// - send: The function responsible for sending the block header response to the client. +// +// Returns a function that can be used as a callback for block header updates. +// +// This function is designed to be used as a callback for block header updates in a subscription. +// It takes a block header, processes it, and sends the corresponding response to the client using the provided send function. +// +// Expected errors during normal operation: +// - codes.Internal: If could not decode the signer indices from the given block header, could not convert a block header to a message or the stream could not send a response. +func (h *Handler) handleBlockHeadersResponse(send sendSubscribeBlockHeadersResponseFunc) func(*flow.Header) error { + return func(header *flow.Header) error { + signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(header) + if err != nil { + return rpc.ConvertError(err, "could not decode the signer indices from the given block header", codes.Internal) // the block was retrieved from local storage - so no errors are expected + } + + msgHeader, err := convert.BlockHeaderToMessage(header, signerIDs) + if err != nil { + return rpc.ConvertError(err, "could not convert block header to message", codes.Internal) + } + + err = send(&access.SubscribeBlockHeadersResponse{ + Header: msgHeader, + }) + if err != nil { + return rpc.ConvertError(err, "could not send response", codes.Internal) + } + + return nil + } +} + +// SubscribeBlockDigestsFromStartBlockID streams finalized or sealed lightweight block starting at the requested block id. +// It takes a SubscribeBlockDigestsFromStartBlockIDRequest and an AccessAPI_SubscribeBlockDigestsFromStartBlockIDServer stream as input. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if invalid startBlockID provided or unknown block status provided, +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block to message or could not send response. +func (h *Handler) SubscribeBlockDigestsFromStartBlockID(request *access.SubscribeBlockDigestsFromStartBlockIDRequest, stream access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + startBlockID, blockStatus, err := h.getSubscriptionDataFromStartBlockID(request.GetStartBlockId(), request.GetBlockStatus()) + if err != nil { + return err + } + + sub := h.api.SubscribeBlockDigestsFromStartBlockID(stream.Context(), startBlockID, blockStatus) + return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send)) +} + +// SubscribeBlockDigestsFromStartHeight handles subscription requests for lightweight blocks started from block height. +// It takes a SubscribeBlockDigestsFromStartHeightRequest and an AccessAPI_SubscribeBlockDigestsFromStartHeightServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block to message or could not send response. +func (h *Handler) SubscribeBlockDigestsFromStartHeight(request *access.SubscribeBlockDigestsFromStartHeightRequest, stream access.AccessAPI_SubscribeBlockDigestsFromStartHeightServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + blockStatus := convert.MessageToBlockStatus(request.GetBlockStatus()) + err := checkBlockStatus(blockStatus) + if err != nil { + return err + } + + sub := h.api.SubscribeBlockDigestsFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus) + return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send)) +} + +// SubscribeBlockDigestsFromLatest handles subscription requests for lightweight block started from latest sealed block. +// It takes a SubscribeBlockDigestsFromLatestRequest and an AccessAPI_SubscribeBlockDigestsFromLatestServer stream as input. +// The handler manages the subscription to block updates and sends the subscribed block header information +// to the client via the provided stream. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if unknown block status provided. +// - codes.ResourceExhausted - if the maximum number of streams is reached. +// - codes.Internal - if stream encountered an error, if stream got unexpected response or could not convert block to message or could not send response. +func (h *Handler) SubscribeBlockDigestsFromLatest(request *access.SubscribeBlockDigestsFromLatestRequest, stream access.AccessAPI_SubscribeBlockDigestsFromLatestServer) error { + // check if the maximum number of streams is reached + if h.StreamCount.Load() >= h.MaxStreams { + return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") + } + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) + + blockStatus := convert.MessageToBlockStatus(request.GetBlockStatus()) + err := checkBlockStatus(blockStatus) + if err != nil { + return err + } + + sub := h.api.SubscribeBlockDigestsFromLatest(stream.Context(), blockStatus) + return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send)) +} + +// handleBlockDigestsResponse handles the subscription to block updates and sends +// the subscribed block digest information to the client via the provided stream. +// +// Parameters: +// - send: The function responsible for sending the block digest response to the client. +// +// Returns a function that can be used as a callback for block digest updates. +// +// This function is designed to be used as a callback for block digest updates in a subscription. +// It takes a block digest, processes it, and sends the corresponding response to the client using the provided send function. +// +// Expected errors during normal operation: +// - codes.Internal: if the stream cannot send a response. +func (h *Handler) handleBlockDigestsResponse(send sendSubscribeBlockDigestsResponseFunc) func(*flow.BlockDigest) error { + return func(blockDigest *flow.BlockDigest) error { + err := send(&access.SubscribeBlockDigestsResponse{ + BlockId: convert.IdentifierToMessage(blockDigest.ID()), + BlockHeight: blockDigest.Height, + BlockTimestamp: timestamppb.New(blockDigest.Timestamp), + }) + if err != nil { + return rpc.ConvertError(err, "could not send response", codes.Internal) + } + + return nil + } +} + +// getSubscriptionDataFromStartBlockID processes subscription start data from start block id. +// It takes a union representing the start block id and a BlockStatus from the entities package. +// Performs validation of input data and returns it in expected format for further processing. +// +// Returns: +// - flow.Identifier: The start block id for searching. +// - flow.BlockStatus: Block status. +// - error: An error indicating the result of the operation, if any. +// +// Expected errors during normal operation: +// - codes.InvalidArgument: If blockStatus is flow.BlockStatusUnknown, or startBlockID could not convert to flow.Identifier. +func (h *Handler) getSubscriptionDataFromStartBlockID(msgBlockId []byte, msgBlockStatus entities.BlockStatus) (flow.Identifier, flow.BlockStatus, error) { + startBlockID, err := convert.BlockID(msgBlockId) + if err != nil { + return flow.ZeroID, flow.BlockStatusUnknown, err + } + + blockStatus := convert.MessageToBlockStatus(msgBlockStatus) + err = checkBlockStatus(blockStatus) + if err != nil { + return flow.ZeroID, flow.BlockStatusUnknown, err + } + + return startBlockID, blockStatus, nil +} + +func (h *Handler) SendAndSubscribeTransactionStatuses(_ *access.SendAndSubscribeTransactionStatusesRequest, _ access.AccessAPI_SendAndSubscribeTransactionStatusesServer) error { + // not implemented + return nil +} + func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flow.BlockStatus) (*access.BlockResponse, error) { metadata := h.buildMetadataResponse() @@ -713,7 +1103,7 @@ func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flo if fullResponse { msg, err = convert.BlockToMessage(block, signerIDs) if err != nil { - return nil, err + return nil, rpc.ConvertError(err, "could not convert block to message", codes.Internal) } } else { msg = convert.BlockToMessageLight(block) @@ -736,7 +1126,7 @@ func (h *Handler) blockHeaderResponse(header *flow.Header, status flow.BlockStat msg, err := convert.BlockHeaderToMessage(header, signerIDs) if err != nil { - return nil, err + return nil, rpc.ConvertError(err, "could not convert block header to message", codes.Internal) } return &access.BlockHeaderResponse{ @@ -777,3 +1167,14 @@ func WithBlockSignerDecoder(signerIndicesDecoder hotstuff.BlockSignerDecoder) fu handler.signerIndicesDecoder = signerIndicesDecoder } } + +// checkBlockStatus checks the validity of the provided block status. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if blockStatus is flow.BlockStatusUnknown +func checkBlockStatus(blockStatus flow.BlockStatus) error { + if blockStatus != flow.BlockStatusFinalized && blockStatus != flow.BlockStatusSealed { + return status.Errorf(codes.InvalidArgument, "block status is unknown. Possible variants: BLOCK_FINALIZED, BLOCK_SEALED") + } + return nil +} diff --git a/access/mock/api.go b/access/mock/api.go index 0e9fc6a0919..972143fee9f 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -12,6 +12,8 @@ import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" + + subscription "github.com/onflow/flow-go/engine/access/subscription" ) // API is an autogenerated mock type for the API type @@ -831,6 +833,150 @@ func (_m *API) SendTransaction(ctx context.Context, tx *flow.TransactionBody) er return r0 } +// SubscribeBlockDigestsFromLatest provides a mock function with given fields: ctx, blockStatus +func (_m *API) SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlockDigestsFromStartBlockID provides a mock function with given fields: ctx, startBlockID, blockStatus +func (_m *API) SubscribeBlockDigestsFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, startBlockID, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, startBlockID, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlockDigestsFromStartHeight provides a mock function with given fields: ctx, startHeight, blockStatus +func (_m *API) SubscribeBlockDigestsFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, startHeight, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, uint64, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, startHeight, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlockHeadersFromLatest provides a mock function with given fields: ctx, blockStatus +func (_m *API) SubscribeBlockHeadersFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlockHeadersFromStartBlockID provides a mock function with given fields: ctx, startBlockID, blockStatus +func (_m *API) SubscribeBlockHeadersFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, startBlockID, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, startBlockID, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlockHeadersFromStartHeight provides a mock function with given fields: ctx, startHeight, blockStatus +func (_m *API) SubscribeBlockHeadersFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, startHeight, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, uint64, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, startHeight, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlocksFromLatest provides a mock function with given fields: ctx, blockStatus +func (_m *API) SubscribeBlocksFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlocksFromStartBlockID provides a mock function with given fields: ctx, startBlockID, blockStatus +func (_m *API) SubscribeBlocksFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, startBlockID, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, startBlockID, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeBlocksFromStartHeight provides a mock function with given fields: ctx, startHeight, blockStatus +func (_m *API) SubscribeBlocksFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription { + ret := _m.Called(ctx, startHeight, blockStatus) + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, uint64, flow.BlockStatus) subscription.Subscription); ok { + r0 = rf(ctx, startHeight, blockStatus) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + type mockConstructorTestingTNewAPI interface { mock.TestingT Cleanup(func()) diff --git a/admin/command_runner.go b/admin/command_runner.go index c827fb5ff4c..d0d39fd6c92 100644 --- a/admin/command_runner.go +++ b/admin/command_runner.go @@ -239,6 +239,7 @@ func (r *CommandRunner) runAdminServer(ctx irrecoverable.SignalerContext) error for _, name := range []string{"allocs", "block", "goroutine", "heap", "mutex", "threadcreate"} { mux.HandleFunc(fmt.Sprintf("/debug/pprof/%s", name), pprof.Handler(name).ServeHTTP) } + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) httpServer := &http.Server{ diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 3d128f02c16..16fa42a5d8a 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -37,6 +37,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/ingestion" pingeng "github.com/onflow/flow-go/engine/access/ping" "github.com/onflow/flow-go/engine/access/rest" @@ -46,6 +47,7 @@ import ( rpcConnection "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/access/state_stream" statestreambackend "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" followereng "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/engine/common/requester" synceng "github.com/onflow/flow-go/engine/common/synchronization" @@ -84,7 +86,7 @@ import ( "github.com/onflow/flow-go/network/p2p/conduit" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" - "github.com/onflow/flow-go/network/p2p/subscription" + networkingsubscription "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" relaynet "github.com/onflow/flow-go/network/relay" @@ -201,14 +203,14 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { }, stateStreamConf: statestreambackend.Config{ MaxExecutionDataMsgSize: grpcutils.DefaultMaxMsgSize, - ExecutionDataCacheSize: state_stream.DefaultCacheSize, - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, - MaxGlobalStreams: state_stream.DefaultMaxGlobalStreams, + ExecutionDataCacheSize: subscription.DefaultCacheSize, + ClientSendTimeout: subscription.DefaultSendTimeout, + ClientSendBufferSize: subscription.DefaultSendBufferSize, + MaxGlobalStreams: subscription.DefaultMaxGlobalStreams, EventFilterConfig: state_stream.DefaultEventFilterConfig, - ResponseLimit: state_stream.DefaultResponseLimit, - HeartbeatInterval: state_stream.DefaultHeartbeatInterval, RegisterIDsRequestLimit: state_stream.DefaultRegisterIDsRequestLimit, + ResponseLimit: subscription.DefaultResponseLimit, + HeartbeatInterval: subscription.DefaultHeartbeatInterval, }, stateStreamFilterConf: nil, ExecutionNodeAddress: "localhost:9000", @@ -283,8 +285,8 @@ type FlowAccessNodeBuilder struct { ExecutionIndexerCore *indexer.IndexerCore ScriptExecutor *backend.ScriptExecutor RegistersAsyncStore *execution.RegistersAsyncStore - EventsIndex *backend.EventsIndex - TxResultsIndex *backend.TransactionResultsIndex + EventsIndex *index.EventsIndex + TxResultsIndex *index.TransactionResultsIndex IndexerDependencies *cmd.DependencyList collectionExecutedMetric module.CollectionExecutedMetric @@ -887,6 +889,17 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess useIndex := builder.executionDataIndexingEnabled && eventQueryMode != backend.IndexQueryModeExecutionNodesOnly + executionDataTracker := subscription.NewExecutionDataTracker( + builder.Logger, + node.State, + builder.executionDataConfig.InitialBlockHeight, + node.Storage.Headers, + broadcaster, + highestAvailableHeight, + builder.EventsIndex, + useIndex, + ) + builder.stateStreamBackend, err = statestreambackend.New( node.Logger, builder.stateStreamConf, @@ -897,11 +910,10 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess builder.ExecutionDataStore, executionDataStoreCache, broadcaster, - builder.executionDataConfig.InitialBlockHeight, - highestAvailableHeight, builder.RegistersAsyncStore, builder.EventsIndex, useIndex, + executionDataTracker, ) if err != nil { return nil, fmt.Errorf("could not create state stream backend: %w", err) @@ -915,14 +927,14 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess node.RootChainID, builder.stateStreamGrpcServer, builder.stateStreamBackend, - broadcaster, ) if err != nil { return nil, fmt.Errorf("could not create state stream engine: %w", err) } builder.StateStreamEng = stateStreamEng - execDataDistributor.AddOnExecutionDataReceivedConsumer(builder.StateStreamEng.OnExecutionData) + // setup requester to notify ExecutionDataTracker when new execution data is received + execDataDistributor.AddOnExecutionDataReceivedConsumer(builder.stateStreamBackend.OnExecutionData) return builder.StateStreamEng, nil }) @@ -1526,11 +1538,11 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Module("events index", func(node *cmd.NodeConfig) error { - builder.EventsIndex = backend.NewEventsIndex(builder.Storage.Events) + builder.EventsIndex = index.NewEventsIndex(builder.Storage.Events) return nil }). Module("transaction result index", func(node *cmd.NodeConfig) error { - builder.TxResultsIndex = backend.NewTransactionResultsIndex(builder.Storage.LightTransactionResults) + builder.TxResultsIndex = index.NewTransactionResultsIndex(builder.Storage.LightTransactionResults) return nil }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { @@ -1578,6 +1590,18 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil, fmt.Errorf("event query mode 'compare' is not supported") } + broadcaster := engine.NewBroadcaster() + // create BlockTracker that will track for new blocks (finalized and sealed) and + // handles block-related operations. + blockTracker, err := subscription.NewBlockTracker( + node.State, + builder.FinalizedRootBlock.Header.Height, + node.Storage.Headers, + broadcaster, + ) + if err != nil { + return nil, fmt.Errorf("failed to initialize block tracker: %w", err) + } txResultQueryMode, err := backend.ParseIndexQueryMode(config.BackendConfig.TxResultQueryMode) if err != nil { return nil, fmt.Errorf("could not parse transaction result query mode: %w", err) @@ -1611,9 +1635,16 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { ScriptExecutor: builder.ScriptExecutor, ScriptExecutionMode: scriptExecMode, EventQueryMode: eventQueryMode, - EventsIndex: builder.EventsIndex, - TxResultQueryMode: txResultQueryMode, - TxResultsIndex: builder.TxResultsIndex, + BlockTracker: blockTracker, + SubscriptionParams: backend.SubscriptionParams{ + Broadcaster: broadcaster, + SendTimeout: builder.stateStreamConf.ClientSendTimeout, + ResponseLimit: builder.stateStreamConf.ResponseLimit, + SendBufferSize: int(builder.stateStreamConf.ClientSendBufferSize), + }, + EventsIndex: builder.EventsIndex, + TxResultQueryMode: txResultQueryMode, + TxResultsIndex: builder.TxResultsIndex, }) if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) @@ -1873,7 +1904,7 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri Unicast: builder.FlowConfig.NetworkConfig.Unicast, }). SetBasicResolver(builder.Resolver). - SetSubscriptionFilter(subscription.NewRoleBasedFilter(flow.RoleAccess, builder.IdentityProvider)). + SetSubscriptionFilter(networkingsubscription.NewRoleBasedFilter(flow.RoleAccess, builder.IdentityProvider)). SetConnectionManager(connManager). SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { return dht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), builder.Logger, networkMetrics, dht.AsServer()) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index e33c6e64326..7a83966d241 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -35,7 +35,9 @@ import ( hotstuffvalidator "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" + "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/access/apiproxy" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/rest" restapiproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" "github.com/onflow/flow-go/engine/access/rest/routes" @@ -44,6 +46,7 @@ import ( rpcConnection "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/access/state_stream" statestreambackend "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/follower" synceng "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/engine/execution/computation/query" @@ -57,6 +60,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/chainsync" + "github.com/onflow/flow-go/module/execution" "github.com/onflow/flow-go/module/executiondatasync/execution_data" execdatacache "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" @@ -83,7 +87,7 @@ import ( p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" p2plogging "github.com/onflow/flow-go/network/p2p/logging" - "github.com/onflow/flow-go/network/p2p/subscription" + networkingsubscription "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" @@ -132,6 +136,8 @@ type ObserverServiceConfig struct { registersDBPath string checkpointFile string apiTimeout time.Duration + stateStreamConf statestreambackend.Config + stateStreamFilterConf map[string]int upstreamNodeAddresses []string upstreamNodePublicKeys []string upstreamIdentities flow.IdentitySkeletonList // the identity list of upstream peers the node uses to forward API requests to @@ -141,7 +147,6 @@ type ObserverServiceConfig struct { executionDataDir string executionDataStartHeight uint64 executionDataConfig edrequester.ExecutionDataConfig - executionDataCacheSize uint32 // TODO: remove it when state stream is added } // DefaultObserverServiceConfig defines all the default values for the ObserverServiceConfig @@ -171,6 +176,18 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { MaxMsgSize: grpcutils.DefaultMaxMsgSize, CompressorName: grpcutils.NoCompressor, }, + stateStreamConf: statestreambackend.Config{ + MaxExecutionDataMsgSize: grpcutils.DefaultMaxMsgSize, + ExecutionDataCacheSize: subscription.DefaultCacheSize, + ClientSendTimeout: subscription.DefaultSendTimeout, + ClientSendBufferSize: subscription.DefaultSendBufferSize, + MaxGlobalStreams: subscription.DefaultMaxGlobalStreams, + EventFilterConfig: state_stream.DefaultEventFilterConfig, + ResponseLimit: subscription.DefaultResponseLimit, + HeartbeatInterval: subscription.DefaultHeartbeatInterval, + RegisterIDsRequestLimit: state_stream.DefaultRegisterIDsRequestLimit, + }, + stateStreamFilterConf: nil, rpcMetricsEnabled: false, apiRatelimits: nil, apiBurstlimits: nil, @@ -195,7 +212,6 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { RetryDelay: edrequester.DefaultRetryDelay, MaxRetryDelay: edrequester.DefaultMaxRetryDelay, }, - executionDataCacheSize: state_stream.DefaultCacheSize, } } @@ -206,39 +222,48 @@ type ObserverServiceBuilder struct { *ObserverServiceConfig // components - LibP2PNode p2p.LibP2PNode - FollowerState stateprotocol.FollowerState - SyncCore *chainsync.Core - RpcEng *rpc.Engine - FollowerDistributor *pubsub.FollowerDistributor - Committee hotstuff.DynamicCommittee - Finalized *flow.Header - Pending []*flow.Header - FollowerCore module.HotStuffFollower - ExecutionDataRequester state_synchronization.ExecutionDataRequester - ExecutionIndexer *indexer.Indexer - ExecutionIndexerCore *indexer.IndexerCore - IndexerDependencies *cmd.DependencyList + + LibP2PNode p2p.LibP2PNode + FollowerState stateprotocol.FollowerState + SyncCore *chainsync.Core + RpcEng *rpc.Engine + FollowerDistributor *pubsub.FollowerDistributor + Committee hotstuff.DynamicCommittee + Finalized *flow.Header + Pending []*flow.Header + FollowerCore module.HotStuffFollower + ExecutionIndexer *indexer.Indexer + ExecutionIndexerCore *indexer.IndexerCore + IndexerDependencies *cmd.DependencyList + + ExecutionDataDownloader execution_data.Downloader + ExecutionDataRequester state_synchronization.ExecutionDataRequester + ExecutionDataStore execution_data.ExecutionDataStore + + RegistersAsyncStore *execution.RegistersAsyncStore + EventsIndex *index.EventsIndex // available until after the network has started. Hence, a factory function that needs to be called just before // creating the sync engine SyncEngineParticipantsProviderFactory func() module.IdentifierProvider // engines - FollowerEng *follower.ComplianceEngine - SyncEng *synceng.Engine + FollowerEng *follower.ComplianceEngine + SyncEng *synceng.Engine + StateStreamEng *statestreambackend.Engine // Public network peerID peer.ID RestMetrics *metrics.RestCollector AccessMetrics module.AccessMetrics + // grpc servers - secureGrpcServer *grpcserver.GrpcServer - unsecureGrpcServer *grpcserver.GrpcServer + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer + stateStreamGrpcServer *grpcserver.GrpcServer - ExecutionDataDownloader execution_data.Downloader - ExecutionDataStore execution_data.ExecutionDataStore + stateStreamBackend *statestreambackend.StateStreamBackend } // deriveBootstrapPeerIdentities derives the Flow Identity of the bootstrap peers from the parameters. @@ -630,10 +655,53 @@ func (builder *ObserverServiceBuilder) extraFlags() { "execution-data-max-retry-delay", defaultConfig.executionDataConfig.MaxRetryDelay, "maximum delay for exponential backoff when fetching execution data fails e.g. 5m") - flags.Uint32Var(&builder.executionDataCacheSize, + + // Streaming API + flags.StringVar(&builder.stateStreamConf.ListenAddr, + "state-stream-addr", + defaultConfig.stateStreamConf.ListenAddr, + "the address the state stream server listens on (if empty the server will not be started)") + flags.Uint32Var(&builder.stateStreamConf.ExecutionDataCacheSize, "execution-data-cache-size", - defaultConfig.executionDataCacheSize, + defaultConfig.stateStreamConf.ExecutionDataCacheSize, "block execution data cache size") + flags.Uint32Var(&builder.stateStreamConf.MaxGlobalStreams, + "state-stream-global-max-streams", defaultConfig.stateStreamConf.MaxGlobalStreams, + "global maximum number of concurrent streams") + flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, + "state-stream-max-message-size", + defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, + "maximum size for a gRPC message containing block execution data") + flags.StringToIntVar(&builder.stateStreamFilterConf, + "state-stream-event-filter-limits", + defaultConfig.stateStreamFilterConf, + "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") + flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, + "state-stream-send-timeout", + defaultConfig.stateStreamConf.ClientSendTimeout, + "maximum wait before timing out while sending a response to a streaming client e.g. 30s") + flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, + "state-stream-send-buffer-size", + defaultConfig.stateStreamConf.ClientSendBufferSize, + "maximum number of responses to buffer within a stream") + flags.Float64Var(&builder.stateStreamConf.ResponseLimit, + "state-stream-response-limit", + defaultConfig.stateStreamConf.ResponseLimit, + "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") + flags.Uint64Var(&builder.stateStreamConf.HeartbeatInterval, + "state-stream-heartbeat-interval", + defaultConfig.stateStreamConf.HeartbeatInterval, + "default interval in blocks at which heartbeat messages should be sent. applied when client did not specify a value.") + flags.Uint32Var(&builder.stateStreamConf.RegisterIDsRequestLimit, + "state-stream-max-register-values", + defaultConfig.stateStreamConf.RegisterIDsRequestLimit, + "maximum number of register ids to include in a single request to the GetRegisters endpoint") + + flags.StringVar(&builder.rpcConf.BackendConfig.EventQueryMode, + "event-query-mode", + defaultConfig.rpcConf.BackendConfig.EventQueryMode, + "mode to use when querying events. one of [local-only, execution-nodes-only(default), failover]") + }).ValidateFlags(func() error { if builder.executionDataSyncEnabled { if builder.executionDataConfig.FetchTimeout <= 0 { @@ -652,6 +720,33 @@ func (builder *ObserverServiceBuilder) extraFlags() { return errors.New("execution-data-max-search-ahead must be greater than 0") } } + if builder.stateStreamConf.ListenAddr != "" { + if builder.stateStreamConf.ExecutionDataCacheSize == 0 { + return errors.New("execution-data-cache-size must be greater than 0") + } + if builder.stateStreamConf.ClientSendBufferSize == 0 { + return errors.New("state-stream-send-buffer-size must be greater than 0") + } + if len(builder.stateStreamFilterConf) > 3 { + return errors.New("state-stream-event-filter-limits must have at most 3 keys (EventTypes, Addresses, Contracts)") + } + for key, value := range builder.stateStreamFilterConf { + switch key { + case "EventTypes", "Addresses", "Contracts": + if value <= 0 { + return fmt.Errorf("state-stream-event-filter-limits %s must be greater than 0", key) + } + default: + return errors.New("state-stream-event-filter-limits may only contain the keys EventTypes, Addresses, Contracts") + } + } + if builder.stateStreamConf.ResponseLimit < 0 { + return errors.New("state-stream-response-limit must be greater than or equal to 0") + } + if builder.stateStreamConf.RegisterIDsRequestLimit <= 0 { + return errors.New("state-stream-max-register-values must be greater than 0") + } + } return nil }) @@ -856,8 +951,8 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr Unicast: builder.FlowConfig.NetworkConfig.Unicast, }). SetSubscriptionFilter( - subscription.NewRoleBasedFilter( - subscription.UnstakedRole, builder.IdentityProvider, + networkingsubscription.NewRoleBasedFilter( + networkingsubscription.UnstakedRole, builder.IdentityProvider, ), ). SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { @@ -976,7 +1071,7 @@ func (builder *ObserverServiceBuilder) BuildExecutionSyncComponents() *ObserverS heroCacheCollector = metrics.AccessNodeExecutionDataCacheMetrics(builder.MetricsRegisterer) } - execDataCacheBackend = herocache.NewBlockExecutionData(builder.executionDataCacheSize, builder.Logger, heroCacheCollector) + execDataCacheBackend = herocache.NewBlockExecutionData(builder.stateStreamConf.ExecutionDataCacheSize, builder.Logger, heroCacheCollector) // Execution Data cache that uses a blobstore as the backend (instead of a downloader) // This ensures that it simply returns a not found error if the blob doesn't exist @@ -1195,10 +1290,100 @@ func (builder *ObserverServiceBuilder) BuildExecutionSyncComponents() *ObserverS // setup requester to notify indexer when new execution data is received execDataDistributor.AddOnExecutionDataReceivedConsumer(builder.ExecutionIndexer.OnExecutionData) + err = builder.EventsIndex.Initialize(builder.ExecutionIndexer) + if err != nil { + return nil, err + } + + err = builder.RegistersAsyncStore.Initialize(registers) + if err != nil { + return nil, err + } + return builder.ExecutionIndexer, nil }, builder.IndexerDependencies) } + if builder.stateStreamConf.ListenAddr != "" { + builder.Component("exec state stream engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + for key, value := range builder.stateStreamFilterConf { + switch key { + case "EventTypes": + builder.stateStreamConf.MaxEventTypes = value + case "Addresses": + builder.stateStreamConf.MaxAddresses = value + case "Contracts": + builder.stateStreamConf.MaxContracts = value + } + } + builder.stateStreamConf.RpcMetricsEnabled = builder.rpcMetricsEnabled + + highestAvailableHeight, err := builder.ExecutionDataRequester.HighestConsecutiveHeight() + if err != nil { + return nil, fmt.Errorf("could not get highest consecutive height: %w", err) + } + broadcaster := engine.NewBroadcaster() + + eventQueryMode, err := backend.ParseIndexQueryMode(builder.rpcConf.BackendConfig.EventQueryMode) + if err != nil { + return nil, fmt.Errorf("could not parse event query mode: %w", err) + } + + // use the events index for events if enabled and the node is configured to use it for + // regular event queries + useIndex := builder.executionDataIndexingEnabled && + eventQueryMode != backend.IndexQueryModeExecutionNodesOnly + + executionDataTracker := subscription.NewExecutionDataTracker( + builder.Logger, + node.State, + builder.executionDataConfig.InitialBlockHeight, + node.Storage.Headers, + broadcaster, + highestAvailableHeight, + builder.EventsIndex, + useIndex, + ) + + builder.stateStreamBackend, err = statestreambackend.New( + node.Logger, + builder.stateStreamConf, + node.State, + node.Storage.Headers, + node.Storage.Seals, + node.Storage.Results, + builder.ExecutionDataStore, + executionDataStoreCache, + broadcaster, + builder.RegistersAsyncStore, + builder.EventsIndex, + useIndex, + executionDataTracker, + ) + if err != nil { + return nil, fmt.Errorf("could not create state stream backend: %w", err) + } + + stateStreamEng, err := statestreambackend.NewEng( + node.Logger, + builder.stateStreamConf, + executionDataStoreCache, + node.Storage.Headers, + node.RootChainID, + builder.stateStreamGrpcServer, + builder.stateStreamBackend, + ) + if err != nil { + return nil, fmt.Errorf("could not create state stream engine: %w", err) + } + builder.StateStreamEng = stateStreamEng + + // setup requester to notify ExecutionDataTracker when new execution data is received + execDataDistributor.AddOnExecutionDataReceivedConsumer(builder.stateStreamBackend.OnExecutionData) + + return builder.StateStreamEng, nil + }) + } return builder } @@ -1315,20 +1500,40 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.apiBurstlimits, grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() - builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, - builder.rpcConf.UnsecureGRPCListenAddr, - builder.rpcConf.MaxMsgSize, + builder.stateStreamGrpcServer = grpcserver.NewGrpcServerBuilder( + node.Logger, + builder.stateStreamConf.ListenAddr, + builder.stateStreamConf.MaxExecutionDataMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, - builder.apiBurstlimits).Build() + builder.apiBurstlimits, + grpcserver.WithStreamInterceptor()).Build() + + if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.rpcConf.UnsecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits).Build() + } else { + builder.unsecureGrpcServer = builder.stateStreamGrpcServer + } return nil }) - + builder.Module("async register store", func(node *cmd.NodeConfig) error { + builder.RegistersAsyncStore = execution.NewRegistersAsyncStore() + return nil + }) builder.Module("events storage", func(node *cmd.NodeConfig) error { builder.Storage.Events = bstorage.NewEvents(node.Metrics.Cache, node.DB) return nil }) + builder.Module("events index", func(node *cmd.NodeConfig) error { + builder.EventsIndex = index.NewEventsIndex(builder.Storage.Events) + return nil + }) builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { accessMetrics := builder.AccessMetrics config := builder.rpcConf @@ -1396,7 +1601,6 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { return nil, err } - stateStreamConfig := statestreambackend.Config{} engineBuilder, err := rpc.NewBuilder( node.Logger, node.State, @@ -1409,8 +1613,8 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { restHandler, builder.secureGrpcServer, builder.unsecureGrpcServer, - nil, // state streaming is not supported - stateStreamConfig, + builder.stateStreamBackend, + builder.stateStreamConf, ) if err != nil { return nil, err @@ -1456,10 +1660,15 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { return builder.secureGrpcServer, nil }) - // build unsecure grpc server - builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - return builder.unsecureGrpcServer, nil + builder.Component("state stream unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.stateStreamGrpcServer, nil }) + + if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { + builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.unsecureGrpcServer, nil + }) + } } func loadNetworkingKey(path string) (crypto.PrivateKey, error) { diff --git a/engine/access/access_test.go b/engine/access/access_test.go index 3f9785729d6..6a1593b615f 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -28,6 +28,7 @@ import ( accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/engine/access/rpc/backend" connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" @@ -171,6 +172,7 @@ func (suite *Suite) RunTest( suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me, + subscription.DefaultMaxGlobalStreams, access.WithBlockSignerDecoder(suite.signerIndicesDecoder), ) f(handler, db, all) @@ -338,7 +340,7 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { }) require.NoError(suite.T(), err) - handler := access.NewHandler(bnd, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) + handler := access.NewHandler(bnd, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me, subscription.DefaultMaxGlobalStreams) // Send transaction 1 resp, err := handler.SendTransaction(context.Background(), sendReq1) @@ -660,7 +662,7 @@ func (suite *Suite) TestGetSealedTransaction() { }) require.NoError(suite.T(), err) - handler := access.NewHandler(bnd, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) + handler := access.NewHandler(bnd, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me, subscription.DefaultMaxGlobalStreams) collectionExecutedMetric, err := indexer.NewCollectionExecutedMetricImpl( suite.log, @@ -811,7 +813,7 @@ func (suite *Suite) TestGetTransactionResult() { }) require.NoError(suite.T(), err) - handler := access.NewHandler(bnd, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) + handler := access.NewHandler(bnd, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me, subscription.DefaultMaxGlobalStreams) collectionExecutedMetric, err := indexer.NewCollectionExecutedMetricImpl( suite.log, @@ -1017,7 +1019,7 @@ func (suite *Suite) TestExecuteScript() { }) require.NoError(suite.T(), err) - handler := access.NewHandler(suite.backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) + handler := access.NewHandler(suite.backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me, subscription.DefaultMaxGlobalStreams) // initialize metrics related storage metrics := metrics.NewNoopCollector() diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index e25ea8fc17b..4a19ad7a157 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -3,6 +3,7 @@ package apiproxy import ( "context" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/rs/zerolog" @@ -230,6 +231,56 @@ func (h *FlowAccessAPIRouter) GetExecutionResultByID(context context.Context, re return res, err } +func (h *FlowAccessAPIRouter) SubscribeBlocksFromStartBlockID(req *access.SubscribeBlocksFromStartBlockIDRequest, server access.AccessAPI_SubscribeBlocksFromStartBlockIDServer) error { + // SubscribeBlocksFromStartBlockID is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlocksFromStartBlockID not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlocksFromStartHeight(req *access.SubscribeBlocksFromStartHeightRequest, server access.AccessAPI_SubscribeBlocksFromStartHeightServer) error { + // SubscribeBlocksFromStartHeight is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlocksFromStartHeight not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlocksFromLatest(req *access.SubscribeBlocksFromLatestRequest, server access.AccessAPI_SubscribeBlocksFromLatestServer) error { + // SubscribeBlocksFromLatest is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlocksFromLatest not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlockHeadersFromStartBlockID(req *access.SubscribeBlockHeadersFromStartBlockIDRequest, server access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDServer) error { + // SubscribeBlockHeadersFromStartBlockID is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlockHeadersFromStartBlockID not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlockHeadersFromStartHeight(req *access.SubscribeBlockHeadersFromStartHeightRequest, server access.AccessAPI_SubscribeBlockHeadersFromStartHeightServer) error { + // SubscribeBlockHeadersFromStartHeight is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlockHeadersFromStartHeight not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlockHeadersFromLatest(req *access.SubscribeBlockHeadersFromLatestRequest, server access.AccessAPI_SubscribeBlockHeadersFromLatestServer) error { + // SubscribeBlockHeadersFromLatest is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlockHeadersFromLatest not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlockDigestsFromStartBlockID(req *access.SubscribeBlockDigestsFromStartBlockIDRequest, server access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDServer) error { + // SubscribeBlockDigestsFromStartBlockID is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlockDigestsFromStartBlockID not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlockDigestsFromStartHeight(req *access.SubscribeBlockDigestsFromStartHeightRequest, server access.AccessAPI_SubscribeBlockDigestsFromStartHeightServer) error { + // SubscribeBlockDigestsFromStartHeight is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlockDigestsFromStartHeight not implemented") +} + +func (h *FlowAccessAPIRouter) SubscribeBlockDigestsFromLatest(req *access.SubscribeBlockDigestsFromLatestRequest, server access.AccessAPI_SubscribeBlockDigestsFromLatestServer) error { + // SubscribeBlockDigestsFromLatest is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SubscribeBlockDigestsFromLatest not implemented") +} + +func (h *FlowAccessAPIRouter) SendAndSubscribeTransactionStatuses(req *access.SendAndSubscribeTransactionStatusesRequest, server access.AccessAPI_SendAndSubscribeTransactionStatusesServer) error { + //SendAndSubscribeTransactionStatuses is not implemented for observer yet + return status.Errorf(codes.Unimplemented, "method SendAndSubscribeTransactionStatuses not implemented") +} + // FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers type FlowAccessAPIForwarder struct { *forwarder.Forwarder diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index 9848fed0424..f68e15805b1 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -157,6 +157,7 @@ func (suite *IrrecoverableStateTestSuite) SetupTest() { Log: suite.log, SnapshotHistoryLimit: 0, Communicator: backend.NewNodeCommunicator(false), + BlockTracker: nil, }) suite.Require().NoError(err) diff --git a/engine/access/rpc/backend/event_index_test.go b/engine/access/index/event_index_test.go similarity index 99% rename from engine/access/rpc/backend/event_index_test.go rename to engine/access/index/event_index_test.go index 845a134ddb1..bb8dd9c51d9 100644 --- a/engine/access/rpc/backend/event_index_test.go +++ b/engine/access/index/event_index_test.go @@ -1,4 +1,4 @@ -package backend +package index import ( "bytes" diff --git a/engine/access/rpc/backend/events_index.go b/engine/access/index/events_index.go similarity index 99% rename from engine/access/rpc/backend/events_index.go rename to engine/access/index/events_index.go index f7d95e62f37..c0e9b50507c 100644 --- a/engine/access/rpc/backend/events_index.go +++ b/engine/access/index/events_index.go @@ -1,4 +1,4 @@ -package backend +package index import ( "fmt" diff --git a/engine/access/rpc/backend/transaction_results_indexer.go b/engine/access/index/transaction_results_indexer.go similarity index 99% rename from engine/access/rpc/backend/transaction_results_indexer.go rename to engine/access/index/transaction_results_indexer.go index 68a05b56e4a..fd9e0f85bcf 100644 --- a/engine/access/rpc/backend/transaction_results_indexer.go +++ b/engine/access/index/transaction_results_indexer.go @@ -1,4 +1,4 @@ -package backend +package index import ( "fmt" diff --git a/engine/access/integration_unsecure_grpc_server_test.go b/engine/access/integration_unsecure_grpc_server_test.go index 40f18686f60..5050051de57 100644 --- a/engine/access/integration_unsecure_grpc_server_test.go +++ b/engine/access/integration_unsecure_grpc_server_test.go @@ -19,11 +19,12 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/index" accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" - "github.com/onflow/flow-go/engine/access/state_stream" statestreambackend "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/execution" @@ -46,19 +47,20 @@ import ( // on the same port type SameGRPCPortTestSuite struct { suite.Suite - state *protocol.State - snapshot *protocol.Snapshot - epochQuery *protocol.EpochQuery - log zerolog.Logger - net *network.EngineRegistry - request *module.Requester - collClient *accessmock.AccessAPIClient - execClient *accessmock.ExecutionAPIClient - me *module.Local - chainID flow.ChainID - metrics *metrics.NoopCollector - rpcEng *rpc.Engine - stateStreamEng *statestreambackend.Engine + state *protocol.State + snapshot *protocol.Snapshot + epochQuery *protocol.EpochQuery + log zerolog.Logger + net *network.EngineRegistry + request *module.Requester + collClient *accessmock.AccessAPIClient + execClient *accessmock.ExecutionAPIClient + me *module.Local + chainID flow.ChainID + metrics *metrics.NoopCollector + rpcEng *rpc.Engine + stateStreamEng *statestreambackend.Engine + executionDataTracker subscription.ExecutionDataTracker // storage blocks *storagemock.Blocks @@ -120,7 +122,7 @@ func (suite *SameGRPCPortTestSuite) SetupTest() { suite.broadcaster = engine.NewBroadcaster() - suite.execDataHeroCache = herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, suite.log, metrics.NewNoopCollector()) + suite.execDataHeroCache = herocache.NewBlockExecutionData(subscription.DefaultCacheSize, suite.log, metrics.NewNoopCollector()) suite.execDataCache = cache.NewExecutionDataCache(suite.eds, suite.headers, suite.seals, suite.results, suite.execDataHeroCache) accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) @@ -234,10 +236,23 @@ func (suite *SameGRPCPortTestSuite) SetupTest() { ).Maybe() conf := statestreambackend.Config{ - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, + ClientSendTimeout: subscription.DefaultSendTimeout, + ClientSendBufferSize: subscription.DefaultSendBufferSize, } + eventIndexer := index.NewEventsIndex(suite.events) + + suite.executionDataTracker = subscription.NewExecutionDataTracker( + suite.log, + suite.state, + rootBlock.Header.Height, + suite.headers, + nil, + rootBlock.Header.Height, + eventIndexer, + false, + ) + stateStreamBackend, err := statestreambackend.New( suite.log, conf, @@ -248,11 +263,10 @@ func (suite *SameGRPCPortTestSuite) SetupTest() { nil, suite.execDataCache, nil, - rootBlock.Header.Height, - rootBlock.Header.Height, suite.registers, - backend.NewEventsIndex(suite.events), + eventIndexer, false, + suite.executionDataTracker, ) assert.NoError(suite.T(), err) @@ -265,7 +279,6 @@ func (suite *SameGRPCPortTestSuite) SetupTest() { suite.chainID, suite.unsecureGrpcServer, stateStreamBackend, - nil, ) assert.NoError(suite.T(), err) diff --git a/engine/access/mock/access_api_client.go b/engine/access/mock/access_api_client.go index 496ee06b58c..17630b8367a 100644 --- a/engine/access/mock/access_api_client.go +++ b/engine/access/mock/access_api_client.go @@ -1007,6 +1007,39 @@ func (_m *AccessAPIClient) Ping(ctx context.Context, in *access.PingRequest, opt return r0, r1 } +// SendAndSubscribeTransactionStatuses provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SendAndSubscribeTransactionStatuses(ctx context.Context, in *access.SendAndSubscribeTransactionStatusesRequest, opts ...grpc.CallOption) (access.AccessAPI_SendAndSubscribeTransactionStatusesClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SendAndSubscribeTransactionStatusesClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SendAndSubscribeTransactionStatusesRequest, ...grpc.CallOption) (access.AccessAPI_SendAndSubscribeTransactionStatusesClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SendAndSubscribeTransactionStatusesRequest, ...grpc.CallOption) access.AccessAPI_SendAndSubscribeTransactionStatusesClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SendAndSubscribeTransactionStatusesClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SendAndSubscribeTransactionStatusesRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // SendTransaction provides a mock function with given fields: ctx, in, opts func (_m *AccessAPIClient) SendTransaction(ctx context.Context, in *access.SendTransactionRequest, opts ...grpc.CallOption) (*access.SendTransactionResponse, error) { _va := make([]interface{}, len(opts)) @@ -1040,6 +1073,303 @@ func (_m *AccessAPIClient) SendTransaction(ctx context.Context, in *access.SendT return r0, r1 } +// SubscribeBlockDigestsFromLatest provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlockDigestsFromLatest(ctx context.Context, in *access.SubscribeBlockDigestsFromLatestRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlockDigestsFromLatestClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlockDigestsFromLatestClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockDigestsFromLatestRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlockDigestsFromLatestClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockDigestsFromLatestRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlockDigestsFromLatestClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlockDigestsFromLatestClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlockDigestsFromLatestRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlockDigestsFromStartBlockID provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlockDigestsFromStartBlockID(ctx context.Context, in *access.SubscribeBlockDigestsFromStartBlockIDRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockDigestsFromStartBlockIDRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockDigestsFromStartBlockIDRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlockDigestsFromStartBlockIDRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlockDigestsFromStartHeight provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlockDigestsFromStartHeight(ctx context.Context, in *access.SubscribeBlockDigestsFromStartHeightRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlockDigestsFromStartHeightClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlockDigestsFromStartHeightClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockDigestsFromStartHeightRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlockDigestsFromStartHeightClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockDigestsFromStartHeightRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlockDigestsFromStartHeightClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlockDigestsFromStartHeightClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlockDigestsFromStartHeightRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlockHeadersFromLatest provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlockHeadersFromLatest(ctx context.Context, in *access.SubscribeBlockHeadersFromLatestRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlockHeadersFromLatestClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlockHeadersFromLatestClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockHeadersFromLatestRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlockHeadersFromLatestClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockHeadersFromLatestRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlockHeadersFromLatestClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlockHeadersFromLatestClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlockHeadersFromLatestRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlockHeadersFromStartBlockID provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlockHeadersFromStartBlockID(ctx context.Context, in *access.SubscribeBlockHeadersFromStartBlockIDRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockHeadersFromStartBlockIDRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockHeadersFromStartBlockIDRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlockHeadersFromStartBlockIDRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlockHeadersFromStartHeight provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlockHeadersFromStartHeight(ctx context.Context, in *access.SubscribeBlockHeadersFromStartHeightRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlockHeadersFromStartHeightClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlockHeadersFromStartHeightClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockHeadersFromStartHeightRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlockHeadersFromStartHeightClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlockHeadersFromStartHeightRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlockHeadersFromStartHeightClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlockHeadersFromStartHeightClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlockHeadersFromStartHeightRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlocksFromLatest provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlocksFromLatest(ctx context.Context, in *access.SubscribeBlocksFromLatestRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlocksFromLatestClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlocksFromLatestClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlocksFromLatestRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlocksFromLatestClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlocksFromLatestRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlocksFromLatestClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlocksFromLatestClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlocksFromLatestRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlocksFromStartBlockID provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlocksFromStartBlockID(ctx context.Context, in *access.SubscribeBlocksFromStartBlockIDRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlocksFromStartBlockIDClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlocksFromStartBlockIDClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlocksFromStartBlockIDRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlocksFromStartBlockIDClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlocksFromStartBlockIDRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlocksFromStartBlockIDClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlocksFromStartBlockIDClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlocksFromStartBlockIDRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubscribeBlocksFromStartHeight provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) SubscribeBlocksFromStartHeight(ctx context.Context, in *access.SubscribeBlocksFromStartHeightRequest, opts ...grpc.CallOption) (access.AccessAPI_SubscribeBlocksFromStartHeightClient, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 access.AccessAPI_SubscribeBlocksFromStartHeightClient + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlocksFromStartHeightRequest, ...grpc.CallOption) (access.AccessAPI_SubscribeBlocksFromStartHeightClient, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.SubscribeBlocksFromStartHeightRequest, ...grpc.CallOption) access.AccessAPI_SubscribeBlocksFromStartHeightClient); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPI_SubscribeBlocksFromStartHeightClient) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.SubscribeBlocksFromStartHeightRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewAccessAPIClient interface { mock.TestingT Cleanup(func()) diff --git a/engine/access/mock/access_api_server.go b/engine/access/mock/access_api_server.go index c9545b26450..dcb9e571200 100644 --- a/engine/access/mock/access_api_server.go +++ b/engine/access/mock/access_api_server.go @@ -795,6 +795,20 @@ func (_m *AccessAPIServer) Ping(_a0 context.Context, _a1 *access.PingRequest) (* return r0, r1 } +// SendAndSubscribeTransactionStatuses provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SendAndSubscribeTransactionStatuses(_a0 *access.SendAndSubscribeTransactionStatusesRequest, _a1 access.AccessAPI_SendAndSubscribeTransactionStatusesServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SendAndSubscribeTransactionStatusesRequest, access.AccessAPI_SendAndSubscribeTransactionStatusesServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // SendTransaction provides a mock function with given fields: _a0, _a1 func (_m *AccessAPIServer) SendTransaction(_a0 context.Context, _a1 *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { ret := _m.Called(_a0, _a1) @@ -821,6 +835,132 @@ func (_m *AccessAPIServer) SendTransaction(_a0 context.Context, _a1 *access.Send return r0, r1 } +// SubscribeBlockDigestsFromLatest provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlockDigestsFromLatest(_a0 *access.SubscribeBlockDigestsFromLatestRequest, _a1 access.AccessAPI_SubscribeBlockDigestsFromLatestServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlockDigestsFromLatestRequest, access.AccessAPI_SubscribeBlockDigestsFromLatestServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlockDigestsFromStartBlockID provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlockDigestsFromStartBlockID(_a0 *access.SubscribeBlockDigestsFromStartBlockIDRequest, _a1 access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlockDigestsFromStartBlockIDRequest, access.AccessAPI_SubscribeBlockDigestsFromStartBlockIDServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlockDigestsFromStartHeight provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlockDigestsFromStartHeight(_a0 *access.SubscribeBlockDigestsFromStartHeightRequest, _a1 access.AccessAPI_SubscribeBlockDigestsFromStartHeightServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlockDigestsFromStartHeightRequest, access.AccessAPI_SubscribeBlockDigestsFromStartHeightServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlockHeadersFromLatest provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlockHeadersFromLatest(_a0 *access.SubscribeBlockHeadersFromLatestRequest, _a1 access.AccessAPI_SubscribeBlockHeadersFromLatestServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlockHeadersFromLatestRequest, access.AccessAPI_SubscribeBlockHeadersFromLatestServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlockHeadersFromStartBlockID provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlockHeadersFromStartBlockID(_a0 *access.SubscribeBlockHeadersFromStartBlockIDRequest, _a1 access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlockHeadersFromStartBlockIDRequest, access.AccessAPI_SubscribeBlockHeadersFromStartBlockIDServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlockHeadersFromStartHeight provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlockHeadersFromStartHeight(_a0 *access.SubscribeBlockHeadersFromStartHeightRequest, _a1 access.AccessAPI_SubscribeBlockHeadersFromStartHeightServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlockHeadersFromStartHeightRequest, access.AccessAPI_SubscribeBlockHeadersFromStartHeightServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlocksFromLatest provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlocksFromLatest(_a0 *access.SubscribeBlocksFromLatestRequest, _a1 access.AccessAPI_SubscribeBlocksFromLatestServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlocksFromLatestRequest, access.AccessAPI_SubscribeBlocksFromLatestServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlocksFromStartBlockID provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlocksFromStartBlockID(_a0 *access.SubscribeBlocksFromStartBlockIDRequest, _a1 access.AccessAPI_SubscribeBlocksFromStartBlockIDServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlocksFromStartBlockIDRequest, access.AccessAPI_SubscribeBlocksFromStartBlockIDServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SubscribeBlocksFromStartHeight provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) SubscribeBlocksFromStartHeight(_a0 *access.SubscribeBlocksFromStartHeightRequest, _a1 access.AccessAPI_SubscribeBlocksFromStartHeightServer) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*access.SubscribeBlocksFromStartHeightRequest, access.AccessAPI_SubscribeBlocksFromStartHeightServer) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + type mockConstructorTestingTNewAccessAPIServer interface { mock.TestingT Cleanup(func()) diff --git a/engine/access/rest/routes/subscribe_events.go b/engine/access/rest/routes/subscribe_events.go index a087961cd71..e1aca3bb316 100644 --- a/engine/access/rest/routes/subscribe_events.go +++ b/engine/access/rest/routes/subscribe_events.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" ) // SubscribeEvents create websocket connection and write to it requested events. @@ -13,7 +14,7 @@ func SubscribeEvents( ctx context.Context, request *request.Request, wsController *WebsocketController, -) (state_stream.Subscription, error) { +) (subscription.Subscription, error) { req, err := request.SubscribeEventsRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/routes/test_helpers.go b/engine/access/rest/routes/test_helpers.go index ebe40fa48df..feae66f5bf9 100644 --- a/engine/access/rest/routes/test_helpers.go +++ b/engine/access/rest/routes/test_helpers.go @@ -18,6 +18,7 @@ import ( "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" @@ -94,8 +95,8 @@ var _ http.Hijacker = (*testHijackResponseRecorder)(nil) // Hijack implements the http.Hijacker interface by returning a fakeNetConn and a bufio.ReadWriter // that simulate a hijacked connection. func (w *testHijackResponseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { - br := bufio.NewReaderSize(strings.NewReader(""), state_stream.DefaultSendBufferSize) - bw := bufio.NewWriterSize(&bytes.Buffer{}, state_stream.DefaultSendBufferSize) + br := bufio.NewReaderSize(strings.NewReader(""), subscription.DefaultSendBufferSize) + bw := bufio.NewWriterSize(&bytes.Buffer{}, subscription.DefaultSendBufferSize) w.responseBuff = bytes.NewBuffer(make([]byte, 0)) w.closed = make(chan struct{}, 1) @@ -137,8 +138,8 @@ func executeWsRequest(req *http.Request, stateStreamApi state_stream.API, respon config := backend.Config{ EventFilterConfig: state_stream.DefaultEventFilterConfig, - MaxGlobalStreams: state_stream.DefaultMaxGlobalStreams, - HeartbeatInterval: state_stream.DefaultHeartbeatInterval, + MaxGlobalStreams: subscription.DefaultMaxGlobalStreams, + HeartbeatInterval: subscription.DefaultHeartbeatInterval, } router := NewRouterBuilder(unittest.Logger(), restCollector).AddWsRoutes( diff --git a/engine/access/rest/routes/websocket_handler.go b/engine/access/rest/routes/websocket_handler.go index 221a18ea7b0..f2261baa76f 100644 --- a/engine/access/rest/routes/websocket_handler.go +++ b/engine/access/rest/routes/websocket_handler.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" ) @@ -110,7 +111,7 @@ func (wsController *WebsocketController) wsErrorHandler(err error) { // It listens to the subscription's channel for events and writes them to the WebSocket connection. // If an error occurs or the subscription channel is closed, it handles the error or termination accordingly. // The function uses a ticker to periodically send ping messages to the client to maintain the connection. -func (wsController *WebsocketController) writeEvents(sub state_stream.Subscription) { +func (wsController *WebsocketController) writeEvents(sub subscription.Subscription) { ticker := time.NewTicker(pingPeriod) defer ticker.Stop() @@ -229,7 +230,7 @@ type SubscribeHandlerFunc func( ctx context.Context, request *request.Request, wsController *WebsocketController, -) (state_stream.Subscription, error) +) (subscription.Subscription, error) // WSHandler is websocket handler implementing custom websocket handler function and allows easier handling of errors and // responses as it wraps functionality for handling error and responses outside of endpoint handling. diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 34eabe6a73a..10eeb1a90a6 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -12,7 +12,10 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/cmd/build" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/rpc/connection" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/model/flow" @@ -69,6 +72,7 @@ type Backend struct { backendAccounts backendExecutionResults backendNetwork + backendSubscribeBlocks state protocol.State chainID flow.ChainID @@ -77,7 +81,8 @@ type Backend struct { connFactory connection.ConnectionFactory // cache the response to GetNodeVersionInfo since it doesn't change - nodeInfo *access.NodeVersionInfo + nodeInfo *access.NodeVersionInfo + BlockTracker subscription.BlockTracker } type Params struct { @@ -105,9 +110,19 @@ type Params struct { ScriptExecutor execution.ScriptExecutor ScriptExecutionMode IndexQueryMode EventQueryMode IndexQueryMode - EventsIndex *EventsIndex - TxResultQueryMode IndexQueryMode - TxResultsIndex *TransactionResultsIndex + BlockTracker subscription.BlockTracker + SubscriptionParams SubscriptionParams + + EventsIndex *index.EventsIndex + TxResultQueryMode IndexQueryMode + TxResultsIndex *index.TransactionResultsIndex +} + +type SubscriptionParams struct { + Broadcaster *engine.Broadcaster + SendTimeout time.Duration + ResponseLimit float64 + SendBufferSize int } var _ TransactionErrorMessage = (*Backend)(nil) @@ -155,7 +170,8 @@ func New(params Params) (*Backend, error) { nodeInfo := getNodeVersionInfo(params.State.Params()) b := &Backend{ - state: params.State, + state: params.State, + BlockTracker: params.BlockTracker, // create the sub-backends backendScripts: backendScripts{ log: params.Log, @@ -234,6 +250,17 @@ func New(params Params) (*Backend, error) { headers: params.Headers, snapshotHistoryLimit: params.SnapshotHistoryLimit, }, + backendSubscribeBlocks: backendSubscribeBlocks{ + log: params.Log, + state: params.State, + headers: params.Headers, + blocks: params.Blocks, + broadcaster: params.SubscriptionParams.Broadcaster, + sendTimeout: params.SubscriptionParams.SendTimeout, + responseLimit: params.SubscriptionParams.ResponseLimit, + sendBufferSize: params.SubscriptionParams.SendBufferSize, + blockTracker: params.BlockTracker, + }, collections: params.Collections, executionReceipts: params.ExecutionReceipts, connFactory: params.ConnFactory, diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 20890715533..2928e22aa7a 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" @@ -36,7 +37,7 @@ type backendEvents struct { maxHeightRange uint nodeCommunicator Communicator queryMode IndexQueryMode - eventsIndex *EventsIndex + eventsIndex *index.EventsIndex } // blockMetadata is used to capture information about requested blocks to avoid repeated blockID diff --git a/engine/access/rpc/backend/backend_events_test.go b/engine/access/rpc/backend/backend_events_test.go index 9c4df9f8138..9c30e6d5353 100644 --- a/engine/access/rpc/backend/backend_events_test.go +++ b/engine/access/rpc/backend/backend_events_test.go @@ -18,6 +18,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/entities" execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/onflow/flow-go/engine/access/index" access "github.com/onflow/flow-go/engine/access/mock" connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" "github.com/onflow/flow-go/engine/common/rpc/convert" @@ -47,7 +48,7 @@ type BackendEventsSuite struct { params *protocol.Params rootHeader *flow.Header - eventsIndex *EventsIndex + eventsIndex *index.EventsIndex events *storagemock.Events headers *storagemock.Headers receipts *storagemock.ExecutionReceipts @@ -83,7 +84,7 @@ func (s *BackendEventsSuite) SetupTest() { s.execClient = access.NewExecutionAPIClient(s.T()) s.executionNodes = unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) - s.eventsIndex = NewEventsIndex(s.events) + s.eventsIndex = index.NewEventsIndex(s.events) blockCount := 5 s.blocks = make([]*flow.Block, blockCount) @@ -309,7 +310,7 @@ func (s *BackendEventsSuite) TestGetEvents_HappyPaths() { s.Run(fmt.Sprintf("all from en - %s - %s", tt.encoding.String(), tt.queryMode), func() { events := storagemock.NewEvents(s.T()) - eventsIndex := NewEventsIndex(events) + eventsIndex := index.NewEventsIndex(events) switch tt.queryMode { case IndexQueryModeLocalOnly: @@ -339,7 +340,7 @@ func (s *BackendEventsSuite) TestGetEvents_HappyPaths() { s.Run(fmt.Sprintf("mixed storage & en - %s - %s", tt.encoding.String(), tt.queryMode), func() { events := storagemock.NewEvents(s.T()) - eventsIndex := NewEventsIndex(events) + eventsIndex := index.NewEventsIndex(events) switch tt.queryMode { case IndexQueryModeLocalOnly, IndexQueryModeExecutionNodesOnly: diff --git a/engine/access/rpc/backend/backend_stream_block_digests_test.go b/engine/access/rpc/backend/backend_stream_block_digests_test.go new file mode 100644 index 00000000000..e6df4ddb824 --- /dev/null +++ b/engine/access/rpc/backend/backend_stream_block_digests_test.go @@ -0,0 +1,148 @@ +package backend + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +type BackendBlockDigestSuite struct { + BackendBlocksSuite +} + +func TestBackendBlockDigestSuite(t *testing.T) { + suite.Run(t, new(BackendBlockDigestSuite)) +} + +// SetupTest initializes the test suite with required dependencies. +func (s *BackendBlockDigestSuite) SetupTest() { + s.BackendBlocksSuite.SetupTest() +} + +// TestSubscribeBlockDigestsFromStartBlockID tests the SubscribeBlockDigestsFromStartBlockID method. +func (s *BackendBlockDigestSuite) TestSubscribeBlockDigestsFromStartBlockID() { + s.blockTracker.On( + "GetStartHeightFromBlockID", + mock.AnythingOfType("flow.Identifier"), + ).Return(func(startBlockID flow.Identifier) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromBlockID(startBlockID) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlockDigestsFromStartBlockID(ctx, startValue.(flow.Identifier), blockStatus) + } + + s.subscribe(call, s.requireBlockDigests, s.subscribeFromStartBlockIdTestCases()) +} + +// TestSubscribeBlockDigestsFromStartHeight tests the SubscribeBlockDigestsFromStartHeight method. +func (s *BackendBlockDigestSuite) TestSubscribeBlockDigestsFromStartHeight() { + s.blockTracker.On( + "GetStartHeightFromHeight", + mock.AnythingOfType("uint64"), + ).Return(func(startHeight uint64) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromHeight(startHeight) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlockDigestsFromStartHeight(ctx, startValue.(uint64), blockStatus) + } + + s.subscribe(call, s.requireBlockDigests, s.subscribeFromStartHeightTestCases()) +} + +// TestSubscribeBlockDigestsFromLatest tests the SubscribeBlockDigestsFromLatest method. +func (s *BackendBlockDigestSuite) TestSubscribeBlockDigestsFromLatest() { + s.blockTracker.On( + "GetStartHeightFromLatest", + mock.Anything, + ).Return(func(ctx context.Context) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromLatest(ctx) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlockDigestsFromLatest(ctx, blockStatus) + } + + s.subscribe(call, s.requireBlockDigests, s.subscribeFromLatestTestCases()) +} + +// requireBlockDigests ensures that the received block digest information matches the expected data. +func (s *BackendBlockDigestSuite) requireBlockDigests(v interface{}, expectedBlock *flow.Block) { + actualBlock, ok := v.(*flow.BlockDigest) + require.True(s.T(), ok, "unexpected response type: %T", v) + + s.Require().Equal(expectedBlock.Header.ID(), actualBlock.ID()) + s.Require().Equal(expectedBlock.Header.Height, actualBlock.Height) + s.Require().Equal(expectedBlock.Header.Timestamp, actualBlock.Timestamp) +} + +// TestSubscribeBlockDigestsHandlesErrors tests error handling scenarios for the SubscribeBlockDigestsFromStartBlockID and SubscribeBlockDigestsFromStartHeight methods in the Backend. +// It ensures that the method correctly returns errors for various invalid input cases. +// +// Test Cases: +// +// 1. Returns error for unindexed start block id: +// - Tests that subscribing to block headers with an unindexed start block ID results in a NotFound error. +// +// 2. Returns error for start height before root height: +// - Validates that attempting to subscribe to block headers with a start height before the root height results in an InvalidArgument error. +// +// 3. Returns error for unindexed start height: +// - Tests that subscribing to block headers with an unindexed start height results in a NotFound error. +// +// Each test case checks for specific error conditions and ensures that the methods responds appropriately. +func (s *BackendBlockDigestSuite) TestSubscribeBlockDigestsHandlesErrors() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // mock block tracker for GetStartHeightFromBlockID + s.blockTracker.On( + "GetStartHeightFromBlockID", + mock.AnythingOfType("flow.Identifier"), + ).Return(func(startBlockID flow.Identifier) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromBlockID(startBlockID) + }, nil) + + s.Run("returns error if unknown start block id is provided", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlockDigestsFromStartBlockID(subCtx, unittest.IdentifierFixture(), flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected %s, got %v: %v", codes.NotFound, status.Code(sub.Err()).String(), sub.Err()) + }) + + // mock block tracker for GetStartHeightFromHeight + s.blockTracker.On( + "GetStartHeightFromHeight", + mock.AnythingOfType("uint64"), + ).Return(func(startHeight uint64) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromHeight(startHeight) + }, nil) + + s.Run("returns error for start height before root height", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlockDigestsFromStartHeight(subCtx, s.rootBlock.Header.Height-1, flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected %s, got %v: %v", codes.InvalidArgument, status.Code(sub.Err()).String(), sub.Err()) + }) + + s.Run("returns error if unknown start height is provided", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlockDigestsFromStartHeight(subCtx, s.blocksArray[len(s.blocksArray)-1].Header.Height+10, flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected %s, got %v: %v", codes.NotFound, status.Code(sub.Err()).String(), sub.Err()) + }) +} diff --git a/engine/access/rpc/backend/backend_stream_block_headers_test.go b/engine/access/rpc/backend/backend_stream_block_headers_test.go new file mode 100644 index 00000000000..764187a7fc9 --- /dev/null +++ b/engine/access/rpc/backend/backend_stream_block_headers_test.go @@ -0,0 +1,148 @@ +package backend + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +type BackendBlockHeadersSuite struct { + BackendBlocksSuite +} + +func TestBackendBlockHeadersSuite(t *testing.T) { + suite.Run(t, new(BackendBlockHeadersSuite)) +} + +// SetupTest initializes the test suite with required dependencies. +func (s *BackendBlockHeadersSuite) SetupTest() { + s.BackendBlocksSuite.SetupTest() +} + +// TestSubscribeBlockHeadersFromStartBlockID tests the SubscribeBlockHeadersFromStartBlockID method. +func (s *BackendBlockHeadersSuite) TestSubscribeBlockHeadersFromStartBlockID() { + s.blockTracker.On( + "GetStartHeightFromBlockID", + mock.AnythingOfType("flow.Identifier"), + ).Return(func(startBlockID flow.Identifier) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromBlockID(startBlockID) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlockHeadersFromStartBlockID(ctx, startValue.(flow.Identifier), blockStatus) + } + + s.subscribe(call, s.requireBlockHeaders, s.subscribeFromStartBlockIdTestCases()) +} + +// TestSubscribeBlockHeadersFromStartHeight tests the SubscribeBlockHeadersFromStartHeight method. +func (s *BackendBlockHeadersSuite) TestSubscribeBlockHeadersFromStartHeight() { + s.blockTracker.On( + "GetStartHeightFromHeight", + mock.AnythingOfType("uint64"), + ).Return(func(startHeight uint64) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromHeight(startHeight) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlockHeadersFromStartHeight(ctx, startValue.(uint64), blockStatus) + } + + s.subscribe(call, s.requireBlockHeaders, s.subscribeFromStartHeightTestCases()) +} + +// TestSubscribeBlockHeadersFromLatest tests the SubscribeBlockHeadersFromLatest method. +func (s *BackendBlockHeadersSuite) TestSubscribeBlockHeadersFromLatest() { + s.blockTracker.On( + "GetStartHeightFromLatest", + mock.Anything, + ).Return(func(ctx context.Context) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromLatest(ctx) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlockHeadersFromLatest(ctx, blockStatus) + } + + s.subscribe(call, s.requireBlockHeaders, s.subscribeFromLatestTestCases()) +} + +// requireBlockHeaders ensures that the received block header information matches the expected data. +func (s *BackendBlockHeadersSuite) requireBlockHeaders(v interface{}, expectedBlock *flow.Block) { + actualHeader, ok := v.(*flow.Header) + require.True(s.T(), ok, "unexpected response type: %T", v) + + s.Require().Equal(expectedBlock.Header.Height, actualHeader.Height) + s.Require().Equal(expectedBlock.Header.ID(), actualHeader.ID()) + s.Require().Equal(*expectedBlock.Header, *actualHeader) +} + +// TestSubscribeBlockHeadersHandlesErrors tests error handling scenarios for the SubscribeBlockHeadersFromStartBlockID and SubscribeBlockHeadersFromStartHeight methods in the Backend. +// It ensures that the method correctly returns errors for various invalid input cases. +// +// Test Cases: +// +// 1. Returns error for unindexed start block id: +// - Tests that subscribing to block headers with an unindexed start block ID results in a NotFound error. +// +// 2. Returns error for start height before root height: +// - Validates that attempting to subscribe to block headers with a start height before the root height results in an InvalidArgument error. +// +// 3. Returns error for unindexed start height: +// - Tests that subscribing to block headers with an unindexed start height results in a NotFound error. +// +// Each test case checks for specific error conditions and ensures that the methods responds appropriately. +func (s *BackendBlockHeadersSuite) TestSubscribeBlockHeadersHandlesErrors() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // mock block tracker for GetStartHeightFromBlockID + s.blockTracker.On( + "GetStartHeightFromBlockID", + mock.AnythingOfType("flow.Identifier"), + ).Return(func(startBlockID flow.Identifier) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromBlockID(startBlockID) + }, nil) + + s.Run("returns error for unknown start block id is provided", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlockHeadersFromStartBlockID(subCtx, unittest.IdentifierFixture(), flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected %s, got %v: %v", codes.NotFound, status.Code(sub.Err()).String(), sub.Err()) + }) + + // mock block tracker for GetStartHeightFromHeight + s.blockTracker.On( + "GetStartHeightFromHeight", + mock.AnythingOfType("uint64"), + ).Return(func(startHeight uint64) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromHeight(startHeight) + }, nil) + + s.Run("returns error if start height before root height", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlockHeadersFromStartHeight(subCtx, s.rootBlock.Header.Height-1, flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected %s, got %v: %v", codes.InvalidArgument, status.Code(sub.Err()).String(), sub.Err()) + }) + + s.Run("returns error for unknown start height is provided", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlockHeadersFromStartHeight(subCtx, s.blocksArray[len(s.blocksArray)-1].Header.Height+10, flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected %s, got %v: %v", codes.NotFound, status.Code(sub.Err()).String(), sub.Err()) + }) +} diff --git a/engine/access/rpc/backend/backend_stream_blocks.go b/engine/access/rpc/backend/backend_stream_blocks.go new file mode 100644 index 00000000000..a2ce866137b --- /dev/null +++ b/engine/access/rpc/backend/backend_stream_blocks.go @@ -0,0 +1,357 @@ +package backend + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/logging" +) + +// backendSubscribeBlocks is a struct representing a backend implementation for subscribing to blocks. +type backendSubscribeBlocks struct { + log zerolog.Logger + state protocol.State + blocks storage.Blocks + headers storage.Headers + broadcaster *engine.Broadcaster + sendTimeout time.Duration + responseLimit float64 + sendBufferSize int + + blockTracker subscription.BlockTracker +} + +// SubscribeBlocksFromStartBlockID subscribes to the finalized or sealed blocks starting at the requested +// start block id, up until the latest available block. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block as it becomes available. +// +// Each block is filtered by the provided block status, and only +// those blocks that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - startBlockID: The identifier of the starting block. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlocksFromStartBlockID will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlocksFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromStartBlockID(ctx, startBlockID, b.getBlockResponse(blockStatus)) +} + +// SubscribeBlocksFromStartHeight subscribes to the finalized or sealed blocks starting at the requested +// start block height, up until the latest available block. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block as it becomes available. +// +// Each block is filtered by the provided block status, and only +// those blocks that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - startHeight: The height of the starting block. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlocksFromStartHeight will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlocksFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromStartHeight(ctx, startHeight, b.getBlockResponse(blockStatus)) +} + +// SubscribeBlocksFromLatest subscribes to the finalized or sealed blocks starting at the latest sealed block, +// up until the latest available block. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block as it becomes available. +// +// Each block is filtered by the provided block status, and only +// those blocks that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlocksFromLatest will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlocksFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromLatest(ctx, b.getBlockResponse(blockStatus)) +} + +// SubscribeBlockHeadersFromStartBlockID streams finalized or sealed block headers starting at the requested +// start block id, up until the latest available block header. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block header as it becomes available. +// +// Each block header are filtered by the provided block status, and only +// those block headers that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - startBlockID: The identifier of the starting block. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlockHeadersFromStartBlockID will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlockHeadersFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromStartBlockID(ctx, startBlockID, b.getBlockHeaderResponse(blockStatus)) +} + +// SubscribeBlockHeadersFromStartHeight streams finalized or sealed block headers starting at the requested +// start block height, up until the latest available block header. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block header as it becomes available. +// +// Each block header are filtered by the provided block status, and only +// those block headers that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - startHeight: The height of the starting block. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlockHeadersFromStartHeight will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlockHeadersFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromStartHeight(ctx, startHeight, b.getBlockHeaderResponse(blockStatus)) +} + +// SubscribeBlockHeadersFromLatest streams finalized or sealed block headers starting at the latest sealed block, +// up until the latest available block header. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block header as it becomes available. +// +// Each block header are filtered by the provided block status, and only +// those block headers that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlockHeadersFromLatest will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlockHeadersFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromLatest(ctx, b.getBlockHeaderResponse(blockStatus)) +} + +// SubscribeBlockDigestsFromStartBlockID streams finalized or sealed lightweight block starting at the requested +// start block id, up until the latest available block. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block as it becomes available. +// +// Each lightweight block are filtered by the provided block status, and only +// those blocks that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - startBlockID: The identifier of the starting block. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlockDigestsFromStartBlockID will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlockDigestsFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromStartBlockID(ctx, startBlockID, b.getBlockDigestResponse(blockStatus)) +} + +// SubscribeBlockDigestsFromStartHeight streams finalized or sealed lightweight block starting at the requested +// start block height, up until the latest available block. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block as it becomes available. +// +// Each lightweight block are filtered by the provided block status, and only +// those blocks that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - startHeight: The height of the starting block. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlockDigestsFromStartHeight will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlockDigestsFromStartHeight(ctx context.Context, startHeight uint64, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromStartHeight(ctx, startHeight, b.getBlockDigestResponse(blockStatus)) +} + +// SubscribeBlockDigestsFromLatest streams finalized or sealed lightweight block starting at the latest sealed block, +// up until the latest available block. Once the latest is +// reached, the stream will remain open and responses are sent for each new +// block as it becomes available. +// +// Each lightweight block are filtered by the provided block status, and only +// those blocks that match the status are returned. +// +// Parameters: +// - ctx: Context for the operation. +// - blockStatus: The status of the block, which could be only BlockStatusSealed or BlockStatusFinalized. +// +// If invalid parameters will be supplied SubscribeBlockDigestsFromLatest will return a failed subscription. +func (b *backendSubscribeBlocks) SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription { + return b.subscribeFromLatest(ctx, b.getBlockDigestResponse(blockStatus)) +} + +// subscribeFromStartBlockID is common method that allows clients to subscribe starting at the requested start block id. +// +// Parameters: +// - ctx: Context for the operation. +// - startBlockID: The identifier of the starting block. +// - getData: The callback used by subscriptions to retrieve data information for the specified height and block status. +// +// If invalid parameters are supplied, subscribeFromStartBlockID will return a failed subscription. +func (b *backendSubscribeBlocks) subscribeFromStartBlockID(ctx context.Context, startBlockID flow.Identifier, getData subscription.GetDataByHeightFunc) subscription.Subscription { + nextHeight, err := b.blockTracker.GetStartHeightFromBlockID(startBlockID) + if err != nil { + return subscription.NewFailedSubscription(err, "could not get start height from block id") + } + return b.subscribe(ctx, nextHeight, getData) +} + +// subscribeFromStartHeight is common method that allows clients to subscribe starting at the requested start block height. +// +// Parameters: +// - ctx: Context for the operation. +// - startHeight: The height of the starting block. +// - getData: The callback used by subscriptions to retrieve data information for the specified height and block status. +// +// If invalid parameters are supplied, subscribeFromStartHeight will return a failed subscription. +func (b *backendSubscribeBlocks) subscribeFromStartHeight(ctx context.Context, startHeight uint64, getData subscription.GetDataByHeightFunc) subscription.Subscription { + nextHeight, err := b.blockTracker.GetStartHeightFromHeight(startHeight) + if err != nil { + return subscription.NewFailedSubscription(err, "could not get start height from block height") + } + return b.subscribe(ctx, nextHeight, getData) +} + +// subscribeFromLatest is common method that allows clients to subscribe starting at the latest sealed block. +// +// Parameters: +// - ctx: Context for the operation. +// - getData: The callback used by subscriptions to retrieve data information for the specified height and block status. +// +// No errors are expected during normal operation. +func (b *backendSubscribeBlocks) subscribeFromLatest(ctx context.Context, getData subscription.GetDataByHeightFunc) subscription.Subscription { + nextHeight, err := b.blockTracker.GetStartHeightFromLatest(ctx) + if err != nil { + return subscription.NewFailedSubscription(err, "could not get start height from latest") + } + return b.subscribe(ctx, nextHeight, getData) +} + +// subscribe is common method that allows clients to subscribe to different types of data. +// +// Parameters: +// - ctx: The context for the subscription. +// - nextHeight: The height of the starting block. +// - getData: The callback used by subscriptions to retrieve data information for the specified height and block status. +// +// No errors are expected during normal operation. +func (b *backendSubscribeBlocks) subscribe(ctx context.Context, nextHeight uint64, getData subscription.GetDataByHeightFunc) subscription.Subscription { + sub := subscription.NewHeightBasedSubscription(b.sendBufferSize, nextHeight, getData) + go subscription.NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) + + return sub +} + +// getBlockResponse returns a GetDataByHeightFunc that retrieves block information for the specified height. +func (b *backendSubscribeBlocks) getBlockResponse(blockStatus flow.BlockStatus) subscription.GetDataByHeightFunc { + return func(_ context.Context, height uint64) (interface{}, error) { + block, err := b.getBlock(height, blockStatus) + if err != nil { + return nil, err + } + + b.log.Trace(). + Hex("block_id", logging.ID(block.ID())). + Uint64("height", height). + Msgf("sending block info") + + return block, nil + } +} + +// getBlockHeaderResponse returns a GetDataByHeightFunc that retrieves block header information for the specified height. +func (b *backendSubscribeBlocks) getBlockHeaderResponse(blockStatus flow.BlockStatus) subscription.GetDataByHeightFunc { + return func(_ context.Context, height uint64) (interface{}, error) { + header, err := b.getBlockHeader(height, blockStatus) + if err != nil { + return nil, err + } + + b.log.Trace(). + Hex("block_id", logging.ID(header.ID())). + Uint64("height", height). + Msgf("sending block header info") + + return header, nil + } +} + +// getBlockDigestResponse returns a GetDataByHeightFunc that retrieves lightweight block information for the specified height. +func (b *backendSubscribeBlocks) getBlockDigestResponse(blockStatus flow.BlockStatus) subscription.GetDataByHeightFunc { + return func(_ context.Context, height uint64) (interface{}, error) { + header, err := b.getBlockHeader(height, blockStatus) + if err != nil { + return nil, err + } + + b.log.Trace(). + Hex("block_id", logging.ID(header.ID())). + Uint64("height", height). + Msgf("sending lightweight block info") + + return flow.NewBlockDigest(header.ID(), header.Height, header.Timestamp), nil + } +} + +// getBlockHeader returns the block header for the given block height. +// Expected errors during normal operation: +// - storage.ErrNotFound: block for the given block height is not available. +func (b *backendSubscribeBlocks) getBlockHeader(height uint64, expectedBlockStatus flow.BlockStatus) (*flow.Header, error) { + err := b.validateHeight(height, expectedBlockStatus) + if err != nil { + return nil, err + } + + // since we are querying a finalized or sealed block header, we can use the height index and save an ID computation + header, err := b.headers.ByHeight(height) + if err != nil { + return nil, err + } + + return header, nil +} + +// getBlock returns the block for the given block height. +// Expected errors during normal operation: +// - storage.ErrNotFound: block for the given block height is not available. +func (b *backendSubscribeBlocks) getBlock(height uint64, expectedBlockStatus flow.BlockStatus) (*flow.Block, error) { + err := b.validateHeight(height, expectedBlockStatus) + if err != nil { + return nil, err + } + + // since we are querying a finalized or sealed block, we can use the height index and save an ID computation + block, err := b.blocks.ByHeight(height) + if err != nil { + return nil, err + } + + return block, nil +} + +// validateHeight checks if the given block height is valid and available based on the expected block status. +// Expected errors during normal operation: +// - storage.ErrNotFound: block for the given block height is not available. +func (b *backendSubscribeBlocks) validateHeight(height uint64, expectedBlockStatus flow.BlockStatus) error { + highestHeight, err := b.blockTracker.GetHighestHeight(expectedBlockStatus) + if err != nil { + return fmt.Errorf("could not get highest available height: %w", err) + } + + // fail early if no notification has been received for the given block height. + // note: it's possible for the data to exist in the data store before the notification is + // received. this ensures a consistent view is available to all streams. + if height > highestHeight { + return fmt.Errorf("block %d is not available yet: %w", height, storage.ErrNotFound) + } + + return nil +} diff --git a/engine/access/rpc/backend/backend_stream_blocks_test.go b/engine/access/rpc/backend/backend_stream_blocks_test.go new file mode 100644 index 00000000000..6d56e00df81 --- /dev/null +++ b/engine/access/rpc/backend/backend_stream_blocks_test.go @@ -0,0 +1,501 @@ +package backend + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine" + connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" + "github.com/onflow/flow-go/engine/access/subscription" + subscriptionmock "github.com/onflow/flow-go/engine/access/subscription/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + protocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/storage" + storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" + "github.com/onflow/flow-go/utils/unittest/mocks" +) + +// BackendBlocksSuite is a test suite for the backendBlocks functionality related to blocks subscription. +// It utilizes the suite to organize and structure test code. +type BackendBlocksSuite struct { + suite.Suite + + state *protocol.State + snapshot *protocol.Snapshot + log zerolog.Logger + + blocks *storagemock.Blocks + headers *storagemock.Headers + blockTracker *subscriptionmock.BlockTracker + blockTrackerReal subscription.BlockTracker + + connectionFactory *connectionmock.ConnectionFactory + + chainID flow.ChainID + + broadcaster *engine.Broadcaster + blocksArray []*flow.Block + blockMap map[uint64]*flow.Block + rootBlock flow.Block + + backend *Backend +} + +// testType represents a test scenario for subscribing +type testType struct { + name string + highestBackfill int + startValue interface{} + blockStatus flow.BlockStatus +} + +func TestBackendBlocksSuite(t *testing.T) { + suite.Run(t, new(BackendBlocksSuite)) +} + +// SetupTest initializes the test suite with required dependencies. +func (s *BackendBlocksSuite) SetupTest() { + s.log = zerolog.New(zerolog.NewConsoleWriter()) + s.state = new(protocol.State) + s.snapshot = new(protocol.Snapshot) + header := unittest.BlockHeaderFixture() + + params := new(protocol.Params) + params.On("SporkID").Return(unittest.IdentifierFixture(), nil) + params.On("ProtocolVersion").Return(uint(unittest.Uint64InRange(10, 30)), nil) + params.On("SporkRootBlockHeight").Return(header.Height, nil) + params.On("SealedRoot").Return(header, nil) + s.state.On("Params").Return(params) + + s.blocks = new(storagemock.Blocks) + s.headers = new(storagemock.Headers) + s.chainID = flow.Testnet + s.connectionFactory = connectionmock.NewConnectionFactory(s.T()) + s.blockTracker = subscriptionmock.NewBlockTracker(s.T()) + + s.broadcaster = engine.NewBroadcaster() + + blockCount := 5 + s.blockMap = make(map[uint64]*flow.Block, blockCount) + s.blocksArray = make([]*flow.Block, 0, blockCount) + + // generate blockCount consecutive blocks with associated seal, result and execution data + s.rootBlock = unittest.BlockFixture() + parent := s.rootBlock.Header + s.blockMap[s.rootBlock.Header.Height] = &s.rootBlock + + for i := 0; i < blockCount; i++ { + block := unittest.BlockWithParentFixture(parent) + // update for next iteration + parent = block.Header + + s.blocksArray = append(s.blocksArray, block) + s.blockMap[block.Header.Height] = block + } + + s.headers.On("ByBlockID", mock.AnythingOfType("flow.Identifier")).Return( + func(blockID flow.Identifier) (*flow.Header, error) { + for _, block := range s.blockMap { + if block.ID() == blockID { + return block.Header, nil + } + } + return nil, storage.ErrNotFound + }, + ).Maybe() + + s.headers.On("ByHeight", mock.AnythingOfType("uint64")).Return( + mocks.ConvertStorageOutput( + mocks.StorageMapGetter(s.blockMap), + func(block *flow.Block) *flow.Header { return block.Header }, + ), + ).Maybe() + + s.blocks.On("ByHeight", mock.AnythingOfType("uint64")).Return( + mocks.StorageMapGetter(s.blockMap), + ).Maybe() + + s.snapshot.On("Head").Return(s.rootBlock.Header, nil).Twice() + s.state.On("Final").Return(s.snapshot, nil).Maybe() + s.state.On("Sealed").Return(s.snapshot, nil).Maybe() + + var err error + s.backend, err = New(s.backendParams()) + require.NoError(s.T(), err) + + // create real block tracker to use GetStartHeight from it, instead of mocking + s.blockTrackerReal, err = subscription.NewBlockTracker( + s.state, + s.rootBlock.Header.Height, + s.headers, + s.broadcaster, + ) + require.NoError(s.T(), err) +} + +// backendParams returns the Params configuration for the backend. +func (s *BackendBlocksSuite) backendParams() Params { + return Params{ + State: s.state, + Blocks: s.blocks, + Headers: s.headers, + ChainID: s.chainID, + MaxHeightRange: DefaultMaxHeightRange, + SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, + AccessMetrics: metrics.NewNoopCollector(), + Log: s.log, + TxErrorMessagesCacheSize: 1000, + SubscriptionParams: SubscriptionParams{ + SendTimeout: subscription.DefaultSendTimeout, + SendBufferSize: subscription.DefaultSendBufferSize, + ResponseLimit: subscription.DefaultResponseLimit, + Broadcaster: s.broadcaster, + }, + BlockTracker: s.blockTracker, + } +} + +// subscribeFromStartBlockIdTestCases generates variations of testType scenarios for subscriptions +// starting from a specified block ID. It is designed to test the subscription functionality when the subscription +// starts from a custom block ID, either sealed or finalized. +func (s *BackendBlocksSuite) subscribeFromStartBlockIdTestCases() []testType { + baseTests := []testType{ + { + name: "happy path - all new blocks", + highestBackfill: -1, // no backfill + startValue: s.rootBlock.ID(), + }, + { + name: "happy path - partial backfill", + highestBackfill: 2, // backfill the first 3 blocks + startValue: s.blocksArray[0].ID(), + }, + { + name: "happy path - complete backfill", + highestBackfill: len(s.blocksArray) - 1, // backfill all blocks + startValue: s.blocksArray[0].ID(), + }, + { + name: "happy path - start from root block by id", + highestBackfill: len(s.blocksArray) - 1, // backfill all blocks + startValue: s.rootBlock.ID(), // start from root block + }, + } + + return s.setupBlockStatusesForTestCases(baseTests) +} + +// subscribeFromStartHeightTestCases generates variations of testType scenarios for subscriptions +// starting from a specified block height. It is designed to test the subscription functionality when the subscription +// starts from a custom height, either sealed or finalized. +func (s *BackendBlocksSuite) subscribeFromStartHeightTestCases() []testType { + baseTests := []testType{ + { + name: "happy path - all new blocks", + highestBackfill: -1, // no backfill + startValue: s.rootBlock.Header.Height, + }, + { + name: "happy path - partial backfill", + highestBackfill: 2, // backfill the first 3 blocks + startValue: s.blocksArray[0].Header.Height, + }, + { + name: "happy path - complete backfill", + highestBackfill: len(s.blocksArray) - 1, // backfill all blocks + startValue: s.blocksArray[0].Header.Height, + }, + { + name: "happy path - start from root block by id", + highestBackfill: len(s.blocksArray) - 1, // backfill all blocks + startValue: s.rootBlock.Header.Height, // start from root block + }, + } + + return s.setupBlockStatusesForTestCases(baseTests) +} + +// subscribeFromLatestTestCases generates variations of testType scenarios for subscriptions +// starting from the latest sealed block. It is designed to test the subscription functionality when the subscription +// starts from the latest available block, either sealed or finalized. +func (s *BackendBlocksSuite) subscribeFromLatestTestCases() []testType { + baseTests := []testType{ + { + name: "happy path - all new blocks", + highestBackfill: -1, // no backfill + }, + { + name: "happy path - partial backfill", + highestBackfill: 2, // backfill the first 3 blocks + }, + { + name: "happy path - complete backfill", + highestBackfill: len(s.blocksArray) - 1, // backfill all blocks + }, + } + + return s.setupBlockStatusesForTestCases(baseTests) +} + +// setupBlockStatusesForTestCases sets up variations for each of the base test cases. +// The function performs the following actions: +// +// 1. Creates variations for each of the provided base test scenarios. +// 2. For each base test, it generates two variations: one for Sealed blocks and one for Finalized blocks. +// 3. Returns a slice of testType containing all variations of test scenarios. +// +// Parameters: +// - baseTests: A slice of testType representing base test scenarios. +func (s *BackendBlocksSuite) setupBlockStatusesForTestCases(baseTests []testType) []testType { + // create variations for each of the base test + tests := make([]testType, 0, len(baseTests)*2) + for _, test := range baseTests { + t1 := test + t1.name = fmt.Sprintf("%s - finalized blocks", test.name) + t1.blockStatus = flow.BlockStatusFinalized + tests = append(tests, t1) + + t2 := test + t2.name = fmt.Sprintf("%s - sealed blocks", test.name) + t2.blockStatus = flow.BlockStatusSealed + tests = append(tests, t2) + } + + return tests +} + +// setupBlockTrackerMock configures a mock for the block tracker based on the provided parameters. +// +// Parameters: +// - blockStatus: The status of the blocks being tracked (Sealed or Finalized). +// - highestHeader: The highest header that the block tracker should report. +func (s *BackendBlocksSuite) setupBlockTrackerMock(blockStatus flow.BlockStatus, highestHeader *flow.Header) { + s.blockTracker.On("GetHighestHeight", mock.Anything).Unset() + s.blockTracker.On("GetHighestHeight", blockStatus).Return(highestHeader.Height, nil) + + if blockStatus == flow.BlockStatusSealed { + s.snapshot.On("Head").Unset() + s.snapshot.On("Head").Return(highestHeader, nil) + } +} + +// TestSubscribeBlocksFromStartBlockID tests the SubscribeBlocksFromStartBlockID method. +func (s *BackendBlocksSuite) TestSubscribeBlocksFromStartBlockID() { + s.blockTracker.On( + "GetStartHeightFromBlockID", + mock.AnythingOfType("flow.Identifier"), + ).Return(func(startBlockID flow.Identifier) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromBlockID(startBlockID) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlocksFromStartBlockID(ctx, startValue.(flow.Identifier), blockStatus) + } + + s.subscribe(call, s.requireBlocks, s.subscribeFromStartBlockIdTestCases()) +} + +// TestSubscribeBlocksFromStartHeight tests the SubscribeBlocksFromStartHeight method. +func (s *BackendBlocksSuite) TestSubscribeBlocksFromStartHeight() { + s.blockTracker.On( + "GetStartHeightFromHeight", + mock.AnythingOfType("uint64"), + ).Return(func(startHeight uint64) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromHeight(startHeight) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlocksFromStartHeight(ctx, startValue.(uint64), blockStatus) + } + + s.subscribe(call, s.requireBlocks, s.subscribeFromStartHeightTestCases()) +} + +// TestSubscribeBlocksFromLatest tests the SubscribeBlocksFromLatest method. +func (s *BackendBlocksSuite) TestSubscribeBlocksFromLatest() { + s.blockTracker.On( + "GetStartHeightFromLatest", + mock.Anything, + ).Return(func(ctx context.Context) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromLatest(ctx) + }, nil) + + call := func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription { + return s.backend.SubscribeBlocksFromLatest(ctx, blockStatus) + } + + s.subscribe(call, s.requireBlocks, s.subscribeFromLatestTestCases()) +} + +// subscribe is the common method with tests the functionality of the subscribe methods in the Backend. +// It covers various scenarios for subscribing, handling backfill, and receiving block updates. +// The test cases include scenarios for both finalized and sealed blocks. +// +// Parameters: +// +// - subscribeFn: A function representing the subscription method to be tested. +// It takes a context, startValue, and blockStatus as parameters +// and returns a subscription.Subscription. +// +// - requireFn: A function responsible for validating that the received information +// matches the expected data. It takes an actual interface{} and an expected *flow.Block as parameters. +// +// - tests: A slice of testType representing different test scenarios for subscriptions. +// +// The function performs the following steps for each test case: +// +// 1. Initializes the test context and cancellation function. +// 2. Iterates through the provided test cases. +// 3. For each test case, sets up a block tracker mock if there are blocks to backfill. +// 4. Mocks the latest sealed block if no start value is provided. +// 5. Subscribes using the provided subscription function. +// 6. Simulates the reception of new blocks and consumes them from the subscription channel. +// 7. Ensures that there are no new messages waiting after all blocks have been processed. +// 8. Cancels the subscription and ensures it shuts down gracefully. +func (s *BackendBlocksSuite) subscribe( + subscribeFn func(ctx context.Context, startValue interface{}, blockStatus flow.BlockStatus) subscription.Subscription, + requireFn func(interface{}, *flow.Block), + tests []testType, +) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, test := range tests { + s.Run(test.name, func() { + // add "backfill" block - blocks that are already in the database before the test starts + // this simulates a subscription on a past block + if test.highestBackfill > 0 { + s.setupBlockTrackerMock(test.blockStatus, s.blocksArray[test.highestBackfill].Header) + } + + subCtx, subCancel := context.WithCancel(ctx) + + // mock latest sealed if no start value provided + if test.startValue == nil { + s.snapshot.On("Head").Unset() + s.snapshot.On("Head").Return(s.rootBlock.Header, nil).Once() + } + + sub := subscribeFn(subCtx, test.startValue, test.blockStatus) + + // loop over all blocks + for i, b := range s.blocksArray { + s.T().Logf("checking block %d %v %d", i, b.ID(), b.Header.Height) + + // simulate new block received. + // all blocks with index <= highestBackfill were already received + if i > test.highestBackfill { + s.setupBlockTrackerMock(test.blockStatus, b.Header) + + s.broadcaster.Publish() + } + + // consume block from subscription + unittest.RequireReturnsBefore(s.T(), func() { + v, ok := <-sub.Channel() + require.True(s.T(), ok, "channel closed while waiting for exec data for block %x %v: err: %v", b.Header.Height, b.ID(), sub.Err()) + + requireFn(v, b) + }, time.Second, fmt.Sprintf("timed out waiting for block %d %v", b.Header.Height, b.ID())) + } + + // make sure there are no new messages waiting. the channel should be opened with nothing waiting + unittest.RequireNeverReturnBefore(s.T(), func() { + <-sub.Channel() + }, 100*time.Millisecond, "timed out waiting for subscription to shutdown") + + // stop the subscription + subCancel() + + // ensure subscription shuts down gracefully + unittest.RequireReturnsBefore(s.T(), func() { + v, ok := <-sub.Channel() + assert.Nil(s.T(), v) + assert.False(s.T(), ok) + assert.ErrorIs(s.T(), sub.Err(), context.Canceled) + }, 100*time.Millisecond, "timed out waiting for subscription to shutdown") + }) + } +} + +// requireBlocks ensures that the received block information matches the expected data. +func (s *BackendBlocksSuite) requireBlocks(v interface{}, expectedBlock *flow.Block) { + actualBlock, ok := v.(*flow.Block) + require.True(s.T(), ok, "unexpected response type: %T", v) + + s.Require().Equal(expectedBlock.Header.Height, actualBlock.Header.Height) + s.Require().Equal(expectedBlock.Header.ID(), actualBlock.Header.ID()) + s.Require().Equal(*expectedBlock, *actualBlock) +} + +// TestSubscribeBlocksHandlesErrors tests error handling scenarios for the SubscribeBlocksFromStartBlockID and SubscribeBlocksFromStartHeight methods in the Backend. +// It ensures that the method correctly returns errors for various invalid input cases. +// +// Test Cases: +// +// 1. Returns error for unindexed start block id: +// - Tests that subscribing to block headers with an unindexed start block ID results in a NotFound error. +// +// 2. Returns error for start height before root height: +// - Validates that attempting to subscribe to block headers with a start height before the root height results in an InvalidArgument error. +// +// 3. Returns error for unindexed start height: +// - Tests that subscribing to block headers with an unindexed start height results in a NotFound error. +// +// Each test case checks for specific error conditions and ensures that the methods responds appropriately. +func (s *BackendBlocksSuite) TestSubscribeBlocksHandlesErrors() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // mock block tracker for SubscribeBlocksFromStartBlockID + s.blockTracker.On( + "GetStartHeightFromBlockID", + mock.AnythingOfType("flow.Identifier"), + ).Return(func(startBlockID flow.Identifier) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromBlockID(startBlockID) + }, nil) + + s.Run("returns error if unknown start block id is provided", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlocksFromStartBlockID(subCtx, unittest.IdentifierFixture(), flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected %s, got %v: %v", codes.NotFound, status.Code(sub.Err()).String(), sub.Err()) + }) + + // mock block tracker for GetStartHeightFromHeight + s.blockTracker.On( + "GetStartHeightFromHeight", + mock.AnythingOfType("uint64"), + ).Return(func(startHeight uint64) (uint64, error) { + return s.blockTrackerReal.GetStartHeightFromHeight(startHeight) + }, nil) + + s.Run("returns error for start height before root height", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlocksFromStartHeight(subCtx, s.rootBlock.Header.Height-1, flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected %s, got %v: %v", codes.InvalidArgument, status.Code(sub.Err()).String(), sub.Err()) + }) + + s.Run("returns error if unknown start height is provided", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeBlocksFromStartHeight(subCtx, s.blocksArray[len(s.blocksArray)-1].Header.Height+10, flow.BlockStatusFinalized) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected %s, got %v: %v", codes.NotFound, status.Code(sub.Err()).String(), sub.Err()) + }) +} diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 30fa3eb5654..a1e6e7fc3dd 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -1289,7 +1289,6 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // TestTransactionPendingToFinalizedStatusTransition tests that the status of transaction changes from Finalized to Expired func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { - ctx := context.Background() collection := unittest.CollectionFixture(1) transactionBody := collection.Transactions[0] @@ -2154,6 +2153,7 @@ func (suite *Suite) defaultBackendParams() Params { AccessMetrics: metrics.NewNoopCollector(), Log: suite.log, TxErrorMessagesCacheSize: 1000, + BlockTracker: nil, TxResultQueryMode: IndexQueryModeExecutionNodesOnly, } } diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index e17e43b373c..e379e905002 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc/status" acc "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/index" connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/fvm/blueprints" @@ -460,7 +461,7 @@ func (suite *Suite) TestLookupTransactionErrorMessageByIndex_HappyPath() { params.ConnFactory = connFactory params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -510,7 +511,7 @@ func (suite *Suite) TestLookupTransactionErrorMessageByIndex_UnknownTransaction( params := suite.defaultBackendParams() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -559,7 +560,7 @@ func (suite *Suite) TestLookupTransactionErrorMessageByIndex_FailedToFetch() { params.ConnFactory = connFactory params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -616,7 +617,7 @@ func (suite *Suite) TestLookupTransactionErrorMessages_HappyPath() { params.ConnFactory = connFactory params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -696,7 +697,7 @@ func (suite *Suite) TestLookupTransactionErrorMessages_HappyPath_NoFailedTxns() params := suite.defaultBackendParams() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -725,7 +726,7 @@ func (suite *Suite) TestLookupTransactionErrorMessages_UnknownTransaction() { params := suite.defaultBackendParams() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -780,7 +781,7 @@ func (suite *Suite) TestLookupTransactionErrorMessages_FailedToFetch() { params.ConnFactory = connFactory params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -1119,11 +1120,11 @@ func (suite *Suite) TestTransactionResultFromStorage() { params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() params.TxResultQueryMode = IndexQueryModeLocalOnly - params.EventsIndex = NewEventsIndex(suite.events) + params.EventsIndex = index.NewEventsIndex(suite.events) err := params.EventsIndex.Initialize(reporter) suite.Require().NoError(err) - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err = params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -1211,11 +1212,11 @@ func (suite *Suite) TestTransactionByIndexFromStorage() { params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() params.TxResultQueryMode = IndexQueryModeLocalOnly - params.EventsIndex = NewEventsIndex(suite.events) + params.EventsIndex = index.NewEventsIndex(suite.events) err := params.EventsIndex.Initialize(reporter) suite.Require().NoError(err) - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err = params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) @@ -1308,11 +1309,11 @@ func (suite *Suite) TestTransactionResultsByBlockIDFromStorage() { params.ConnFactory = connFactory params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - params.EventsIndex = NewEventsIndex(suite.events) + params.EventsIndex = index.NewEventsIndex(suite.events) err := params.EventsIndex.Initialize(reporter) suite.Require().NoError(err) - params.TxResultsIndex = NewTransactionResultsIndex(suite.transactionResults) + params.TxResultsIndex = index.NewTransactionResultsIndex(suite.transactionResults) err = params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) diff --git a/engine/access/rpc/backend/transactions_local_data_provider.go b/engine/access/rpc/backend/transactions_local_data_provider.go index bd8b788e35b..bca27cfa400 100644 --- a/engine/access/rpc/backend/transactions_local_data_provider.go +++ b/engine/access/rpc/backend/transactions_local_data_provider.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc/codes" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -48,8 +49,8 @@ type TransactionsLocalDataProvider struct { state protocol.State collections storage.Collections blocks storage.Blocks - eventsIndex *EventsIndex - txResultsIndex *TransactionResultsIndex + eventsIndex *index.EventsIndex + txResultsIndex *index.TransactionResultsIndex txErrorMessages TransactionErrorMessage systemTxID flow.Identifier } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 4137a1ad976..17f38304dce 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -177,7 +177,22 @@ func (e *Engine) OnFinalizedBlock(block *model.Block) { // No errors expected during normal operations. func (e *Engine) processOnFinalizedBlock(_ *model.Block) error { finalizedHeader := e.finalizedHeaderCache.Get() - return e.backend.ProcessFinalizedBlockHeight(finalizedHeader.Height) + + var err error + // NOTE: The BlockTracker is currently only used by the access node and not by the observer node. + if e.backend.BlockTracker != nil { + err = e.backend.BlockTracker.ProcessOnFinalizedBlock() + if err != nil { + return err + } + } + + err = e.backend.ProcessFinalizedBlockHeight(finalizedHeader.Height) + if err != nil { + return fmt.Errorf("could not process finalized block height %d: %w", finalizedHeader.Height, err) + } + + return nil } // RestApiAddress returns the listen address of the REST API server. diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index 370f3d0fff4..210d0771f18 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -95,9 +95,9 @@ func (builder *RPCEngineBuilder) Build() (*Engine, error) { rpcHandler := builder.rpcHandler if rpcHandler == nil { if builder.signerIndicesDecoder == nil { - rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me) + rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, builder.stateStreamConfig.MaxGlobalStreams) } else { - rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) + rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, builder.stateStreamConfig.MaxGlobalStreams, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) } } accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer.Server, rpcHandler) diff --git a/engine/access/state_stream/backend/backend.go b/engine/access/state_stream/backend/backend.go index db71275e342..a802341c674 100644 --- a/engine/access/state_stream/backend/backend.go +++ b/engine/access/state_stream/backend/backend.go @@ -10,17 +10,14 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/state_stream" - "github.com/onflow/flow-go/engine/common/rpc" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/execution" "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" - "github.com/onflow/flow-go/module/state_synchronization" - "github.com/onflow/flow-go/module/state_synchronization/indexer" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -64,9 +61,10 @@ type Config struct { } type GetExecutionDataFunc func(context.Context, uint64) (*execution_data.BlockExecutionDataEntity, error) -type GetStartHeightFunc func(flow.Identifier, uint64) (uint64, error) type StateStreamBackend struct { + subscription.ExecutionDataTracker + ExecutionDataBackend EventsBackend @@ -78,15 +76,8 @@ type StateStreamBackend struct { execDataStore execution_data.ExecutionDataStore execDataCache *cache.ExecutionDataCache broadcaster *engine.Broadcaster - rootBlockHeight uint64 - rootBlockID flow.Identifier registers *execution.RegistersAsyncStore - indexReporter state_synchronization.IndexReporter registerRequestLimit int - - // highestHeight contains the highest consecutive block height for which we have received a - // new Execution Data notification. - highestHeight counters.StrictMonotonousCounter } func New( @@ -99,21 +90,15 @@ func New( execDataStore execution_data.ExecutionDataStore, execDataCache *cache.ExecutionDataCache, broadcaster *engine.Broadcaster, - rootHeight uint64, - highestAvailableHeight uint64, registers *execution.RegistersAsyncStore, - eventsIndex *backend.EventsIndex, + eventsIndex *index.EventsIndex, useEventsIndex bool, + executionDataTracker subscription.ExecutionDataTracker, ) (*StateStreamBackend, error) { logger := log.With().Str("module", "state_stream_api").Logger() - // cache the root block height and ID for runtime lookups. - rootBlockID, err := headers.BlockIDByHeight(rootHeight) - if err != nil { - return nil, fmt.Errorf("could not get root block ID: %w", err) - } - b := &StateStreamBackend{ + ExecutionDataTracker: executionDataTracker, log: logger, state: state, headers: headers, @@ -122,12 +107,8 @@ func New( execDataStore: execDataStore, execDataCache: execDataCache, broadcaster: broadcaster, - rootBlockHeight: rootHeight, - rootBlockID: rootBlockID, registers: registers, - indexReporter: eventsIndex, registerRequestLimit: int(config.RegisterIDsRequestLimit), - highestHeight: counters.NewMonotonousCounter(highestAvailableHeight), } b.ExecutionDataBackend = ExecutionDataBackend{ @@ -138,7 +119,7 @@ func New( responseLimit: config.ResponseLimit, sendBufferSize: int(config.ClientSendBufferSize), getExecutionData: b.getExecutionData, - getStartHeight: b.getStartHeight, + getStartHeight: b.GetStartHeight, } b.EventsBackend = EventsBackend{ @@ -149,7 +130,7 @@ func New( responseLimit: config.ResponseLimit, sendBufferSize: int(config.ClientSendBufferSize), getExecutionData: b.getExecutionData, - getStartHeight: b.getStartHeight, + getStartHeight: b.GetStartHeight, useIndex: useEventsIndex, eventsIndex: eventsIndex, } @@ -161,10 +142,11 @@ func New( // Expected errors during normal operation: // - storage.ErrNotFound or execution_data.BlobNotFoundError: execution data for the given block height is not available. func (b *StateStreamBackend) getExecutionData(ctx context.Context, height uint64) (*execution_data.BlockExecutionDataEntity, error) { + highestHeight := b.ExecutionDataTracker.GetHighestHeight() // fail early if no notification has been received for the given block height. // note: it's possible for the data to exist in the data store before the notification is // received. this ensures a consistent view is available to all streams. - if height > b.highestHeight.Value() { + if height > highestHeight { return nil, fmt.Errorf("execution data for block %d is not available yet: %w", height, storage.ErrNotFound) } @@ -176,117 +158,6 @@ func (b *StateStreamBackend) getExecutionData(ctx context.Context, height uint64 return execData, nil } -// getStartHeight returns the start height to use when searching. -// Only one of startBlockID and startHeight may be set. Otherwise, an InvalidArgument error is returned. -// If a block is provided and does not exist, a NotFound error is returned. -// If neither startBlockID nor startHeight is provided, the latest sealed block is used. -func (b *StateStreamBackend) getStartHeight(startBlockID flow.Identifier, startHeight uint64) (height uint64, err error) { - // make sure only one of start block ID and start height is provided - if startBlockID != flow.ZeroID && startHeight > 0 { - return 0, status.Errorf(codes.InvalidArgument, "only one of start block ID and start height may be provided") - } - - // ensure that the resolved start height is available - defer func() { - if err == nil { - height, err = b.checkStartHeight(height) - } - }() - - if startBlockID != flow.ZeroID { - return b.startHeightFromBlockID(startBlockID) - } - - if startHeight > 0 { - return b.startHeightFromHeight(startHeight) - } - - // if no start block was provided, use the latest sealed block - header, err := b.state.Sealed().Head() - if err != nil { - return 0, status.Errorf(codes.Internal, "could not get latest sealed block: %v", err) - } - return header.Height, nil -} - -func (b *StateStreamBackend) startHeightFromBlockID(startBlockID flow.Identifier) (uint64, error) { - header, err := b.headers.ByBlockID(startBlockID) - if err != nil { - return 0, rpc.ConvertStorageError(fmt.Errorf("could not get header for block %v: %w", startBlockID, err)) - } - return header.Height, nil -} - -func (b *StateStreamBackend) startHeightFromHeight(startHeight uint64) (uint64, error) { - if startHeight < b.rootBlockHeight { - return 0, status.Errorf(codes.InvalidArgument, "start height must be greater than or equal to the root height %d", b.rootBlockHeight) - } - - header, err := b.headers.ByHeight(startHeight) - if err != nil { - return 0, rpc.ConvertStorageError(fmt.Errorf("could not get header for height %d: %w", startHeight, err)) - } - return header.Height, nil -} - -func (b *StateStreamBackend) checkStartHeight(height uint64) (uint64, error) { - // if the start block is the root block, there will not be an execution data. skip it and - // begin from the next block. - if height == b.rootBlockHeight { - height = b.rootBlockHeight + 1 - } - - if !b.useIndex { - return height, nil - } - - lowestHeight, highestHeight, err := b.getIndexerHeights() - if err != nil { - return 0, err - } - - if height < lowestHeight { - return 0, status.Errorf(codes.InvalidArgument, "start height %d is lower than lowest indexed height %d", height, lowestHeight) - } - - if height > highestHeight { - return 0, status.Errorf(codes.InvalidArgument, "start height %d is higher than highest indexed height %d", height, highestHeight) - } - - return height, nil -} - -// getIndexerHeights returns the lowest and highest indexed block heights -// Expected errors during normal operation: -// - codes.FailedPrecondition: if the index reporter is not ready yet. -// - codes.Internal: if there was any other error getting the heights. -func (b *StateStreamBackend) getIndexerHeights() (uint64, uint64, error) { - lowestHeight, err := b.indexReporter.LowestIndexedHeight() - if err != nil { - if errors.Is(err, storage.ErrHeightNotIndexed) || errors.Is(err, indexer.ErrIndexNotInitialized) { - // the index is not ready yet, but likely will be eventually - return 0, 0, status.Errorf(codes.FailedPrecondition, "failed to get lowest indexed height: %v", err) - } - return 0, 0, rpc.ConvertError(err, "failed to get lowest indexed height", codes.Internal) - } - - highestHeight, err := b.indexReporter.HighestIndexedHeight() - if err != nil { - if errors.Is(err, storage.ErrHeightNotIndexed) || errors.Is(err, indexer.ErrIndexNotInitialized) { - // the index is not ready yet, but likely will be eventually - return 0, 0, status.Errorf(codes.FailedPrecondition, "failed to get highest indexed height: %v", err) - } - return 0, 0, rpc.ConvertError(err, "failed to get highest indexed height", codes.Internal) - } - - return lowestHeight, highestHeight, nil -} - -// setHighestHeight sets the highest height for which execution data is available. -func (b *StateStreamBackend) setHighestHeight(height uint64) bool { - return b.highestHeight.Set(height) -} - // GetRegisterValues returns the register values for the given register IDs at the given block height. func (b *StateStreamBackend) GetRegisterValues(ids flow.RegisterIDs, height uint64) ([]flow.RegisterValue, error) { if len(ids) > b.registerRequestLimit { diff --git a/engine/access/state_stream/backend/backend_events.go b/engine/access/state_stream/backend/backend_events.go index eb4524b81cd..2a6128c3669 100644 --- a/engine/access/state_stream/backend/backend_events.go +++ b/engine/access/state_stream/backend/backend_events.go @@ -8,8 +8,9 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/engine" - rpcbackend "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" @@ -30,27 +31,27 @@ type EventsBackend struct { sendBufferSize int getExecutionData GetExecutionDataFunc - getStartHeight GetStartHeightFunc + getStartHeight subscription.GetStartHeightFunc useIndex bool - eventsIndex *rpcbackend.EventsIndex + eventsIndex *index.EventsIndex } -func (b *EventsBackend) SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter state_stream.EventFilter) state_stream.Subscription { - nextHeight, err := b.getStartHeight(startBlockID, startHeight) +func (b *EventsBackend) SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter state_stream.EventFilter) subscription.Subscription { + nextHeight, err := b.getStartHeight(ctx, startBlockID, startHeight) if err != nil { - return NewFailedSubscription(err, "could not get start height") + return subscription.NewFailedSubscription(err, "could not get start height") } - sub := NewHeightBasedSubscription(b.sendBufferSize, nextHeight, b.getResponseFactory(filter)) + sub := subscription.NewHeightBasedSubscription(b.sendBufferSize, nextHeight, b.getResponseFactory(filter)) - go NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) + go subscription.NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) return sub } -// getResponseFactory returns a function function that returns the event response for a given height. -func (b *EventsBackend) getResponseFactory(filter state_stream.EventFilter) GetDataByHeightFunc { +// getResponseFactory returns a function that returns the event response for a given height. +func (b *EventsBackend) getResponseFactory(filter state_stream.EventFilter) subscription.GetDataByHeightFunc { return func(ctx context.Context, height uint64) (response interface{}, err error) { if b.useIndex { response, err = b.getEventsFromStorage(height, filter) diff --git a/engine/access/state_stream/backend/backend_events_test.go b/engine/access/state_stream/backend/backend_events_test.go index 1b68911f41c..fbd4ce5c4b7 100644 --- a/engine/access/state_stream/backend/backend_events_test.go +++ b/engine/access/state_stream/backend/backend_events_test.go @@ -17,6 +17,7 @@ import ( "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/state_synchronization/indexer" syncmock "github.com/onflow/flow-go/module/state_synchronization/mock" "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/utils/unittest/mocks" @@ -116,12 +117,12 @@ func (s *BackendEventsSuite) runTestSubscribeEvents() { name: "happy path - start from root block by height", highestBackfill: len(s.blocks) - 1, // backfill all blocks startBlockID: flow.ZeroID, - startHeight: s.backend.rootBlockHeight, // start from root block + startHeight: s.rootBlock.Header.Height, // start from root block }, { name: "happy path - start from root block by id", - highestBackfill: len(s.blocks) - 1, // backfill all blocks - startBlockID: s.backend.rootBlockID, // start from root block + highestBackfill: len(s.blocks) - 1, // backfill all blocks + startBlockID: s.rootBlock.Header.ID(), // start from root block startHeight: 0, }, } @@ -155,7 +156,8 @@ func (s *BackendEventsSuite) runTestSubscribeEvents() { // this simulates a subscription on a past block for i := 0; i <= test.highestBackfill; i++ { s.T().Logf("backfilling block %d", i) - s.backend.setHighestHeight(s.blocks[i].Header.Height) + s.executionDataTracker.On("GetHighestHeight"). + Return(s.blocks[i].Header.Height).Maybe() } subCtx, subCancel := context.WithCancel(ctx) @@ -168,7 +170,12 @@ func (s *BackendEventsSuite) runTestSubscribeEvents() { // simulate new exec data received. // exec data for all blocks with index <= highestBackfill were already received if i > test.highestBackfill { - s.backend.setHighestHeight(b.Header.Height) + s.T().Logf("checking block %d %v", i, b.ID()) + + s.executionDataTracker.On("GetHighestHeight").Unset() + s.executionDataTracker.On("GetHighestHeight"). + Return(b.Header.Height).Maybe() + s.broadcaster.Publish() } @@ -228,7 +235,7 @@ func (s *BackendExecutionDataSuite) TestSubscribeEventsHandlesErrors() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() - sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, s.backend.rootBlockHeight-1, state_stream.EventFilter{}) + sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, s.rootBlock.Header.Height-1, state_stream.EventFilter{}) assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected InvalidArgument, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) @@ -251,27 +258,30 @@ func (s *BackendExecutionDataSuite) TestSubscribeEventsHandlesErrors() { assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected NotFound, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) - s.backend.useIndex = true + // Unset GetStartHeight to mock new behavior instead of default one + s.executionDataTracker.On("GetStartHeight", mock.Anything, mock.Anything).Unset() s.Run("returns error for uninitialized index", func() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() + s.executionDataTracker.On("GetStartHeight", subCtx, flow.ZeroID, uint64(0)). + Return(uint64(0), status.Errorf(codes.FailedPrecondition, "failed to get lowest indexed height: %v", indexer.ErrIndexNotInitialized)). + Once() + // Note: eventIndex.Initialize() is not called in this test sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, 0, state_stream.EventFilter{}) assert.Equal(s.T(), codes.FailedPrecondition, status.Code(sub.Err()), "expected FailedPrecondition, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) - reporter := syncmock.NewIndexReporter(s.T()) - reporter.On("LowestIndexedHeight").Return(s.blocks[1].Header.Height, nil) - reporter.On("HighestIndexedHeight").Return(s.blocks[len(s.blocks)-2].Header.Height, nil) - err := s.eventsIndex.Initialize(reporter) - s.Require().NoError(err) - s.Run("returns error for start below lowest indexed", func() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() + s.executionDataTracker.On("GetStartHeight", subCtx, flow.ZeroID, s.blocks[0].Header.Height). + Return(uint64(0), status.Errorf(codes.InvalidArgument, "start height %d is lower than lowest indexed height %d", s.blocks[0].Header.Height, 0)). + Once() + sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, s.blocks[0].Header.Height, state_stream.EventFilter{}) assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected InvalidArgument, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) @@ -280,6 +290,10 @@ func (s *BackendExecutionDataSuite) TestSubscribeEventsHandlesErrors() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() + s.executionDataTracker.On("GetStartHeight", subCtx, flow.ZeroID, s.blocks[len(s.blocks)-1].Header.Height). + Return(uint64(0), status.Errorf(codes.InvalidArgument, "start height %d is higher than highest indexed height %d", s.blocks[len(s.blocks)-1].Header.Height, s.blocks[0].Header.Height)). + Once() + sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, s.blocks[len(s.blocks)-1].Header.Height, state_stream.EventFilter{}) assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected InvalidArgument, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) diff --git a/engine/access/state_stream/backend/backend_executiondata.go b/engine/access/state_stream/backend/backend_executiondata.go index 4a181d33145..e5d6245baec 100644 --- a/engine/access/state_stream/backend/backend_executiondata.go +++ b/engine/access/state_stream/backend/backend_executiondata.go @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/executiondatasync/execution_data" @@ -32,7 +32,7 @@ type ExecutionDataBackend struct { sendBufferSize int getExecutionData GetExecutionDataFunc - getStartHeight GetStartHeightFunc + getStartHeight subscription.GetStartHeightFunc } func (b *ExecutionDataBackend) GetExecutionDataByBlockID(ctx context.Context, blockID flow.Identifier) (*execution_data.BlockExecutionData, error) { @@ -55,15 +55,15 @@ func (b *ExecutionDataBackend) GetExecutionDataByBlockID(ctx context.Context, bl return executionData.BlockExecutionData, nil } -func (b *ExecutionDataBackend) SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startHeight uint64) state_stream.Subscription { - nextHeight, err := b.getStartHeight(startBlockID, startHeight) +func (b *ExecutionDataBackend) SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startHeight uint64) subscription.Subscription { + nextHeight, err := b.getStartHeight(ctx, startBlockID, startHeight) if err != nil { - return NewFailedSubscription(err, "could not get start height") + return subscription.NewFailedSubscription(err, "could not get start height") } - sub := NewHeightBasedSubscription(b.sendBufferSize, nextHeight, b.getResponse) + sub := subscription.NewHeightBasedSubscription(b.sendBufferSize, nextHeight, b.getResponse) - go NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) + go subscription.NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) return sub } diff --git a/engine/access/state_stream/backend/backend_executiondata_test.go b/engine/access/state_stream/backend/backend_executiondata_test.go index 9dc7b57e7df..987c0a697eb 100644 --- a/engine/access/state_stream/backend/backend_executiondata_test.go +++ b/engine/access/state_stream/backend/backend_executiondata_test.go @@ -16,8 +16,10 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" + subscriptionmock "github.com/onflow/flow-go/engine/access/subscription/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/execution" @@ -32,12 +34,14 @@ import ( "github.com/onflow/flow-go/utils/unittest/mocks" ) -var chainID = flow.MonotonicEmulator -var testEventTypes = []flow.EventType{ - unittest.EventTypeFixture(chainID), - unittest.EventTypeFixture(chainID), - unittest.EventTypeFixture(chainID), -} +var ( + chainID = flow.MonotonicEmulator + testEventTypes = []flow.EventType{ + unittest.EventTypeFixture(chainID), + unittest.EventTypeFixture(chainID), + unittest.EventTypeFixture(chainID), + } +) type BackendExecutionDataSuite struct { suite.Suite @@ -51,14 +55,16 @@ type BackendExecutionDataSuite struct { results *storagemock.ExecutionResults registers *storagemock.RegisterIndex registersAsync *execution.RegistersAsyncStore - eventsIndex *backend.EventsIndex + eventsIndex *index.EventsIndex - bs blobs.Blobstore - eds execution_data.ExecutionDataStore - broadcaster *engine.Broadcaster - execDataCache *cache.ExecutionDataCache - execDataHeroCache *herocache.BlockExecutionData - backend *StateStreamBackend + bs blobs.Blobstore + eds execution_data.ExecutionDataStore + broadcaster *engine.Broadcaster + execDataCache *cache.ExecutionDataCache + execDataHeroCache *herocache.BlockExecutionData + executionDataTracker *subscriptionmock.ExecutionDataTracker + backend *StateStreamBackend + executionDataTrackerReal subscription.ExecutionDataTracker blocks []*flow.Block blockEvents map[flow.Identifier][]flow.Event @@ -67,6 +73,8 @@ type BackendExecutionDataSuite struct { sealMap map[flow.Identifier]*flow.Seal resultMap map[flow.Identifier]*flow.ExecutionResult registerID flow.RegisterID + + rootBlock flow.Block } func TestBackendExecutionDataSuite(t *testing.T) { @@ -89,12 +97,13 @@ func (s *BackendExecutionDataSuite) SetupTest() { s.broadcaster = engine.NewBroadcaster() - s.execDataHeroCache = herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, logger, metrics.NewNoopCollector()) + s.execDataHeroCache = herocache.NewBlockExecutionData(subscription.DefaultCacheSize, logger, metrics.NewNoopCollector()) s.execDataCache = cache.NewExecutionDataCache(s.eds, s.headers, s.seals, s.results, s.execDataHeroCache) + s.executionDataTracker = subscriptionmock.NewExecutionDataTracker(s.T()) conf := Config{ - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, + ClientSendTimeout: subscription.DefaultSendTimeout, + ClientSendBufferSize: subscription.DefaultSendBufferSize, RegisterIDsRequestLimit: state_stream.DefaultRegisterIDsRequestLimit, } @@ -109,11 +118,11 @@ func (s *BackendExecutionDataSuite) SetupTest() { s.blocks = make([]*flow.Block, 0, blockCount) // generate blockCount consecutive blocks with associated seal, result and execution data - rootBlock := unittest.BlockFixture() - parent := rootBlock.Header - s.blockMap[rootBlock.Header.Height] = &rootBlock + s.rootBlock = unittest.BlockFixture() + parent := s.rootBlock.Header + s.blockMap[s.rootBlock.Header.Height] = &s.rootBlock - s.T().Logf("Generating %d blocks, root block: %d %s", blockCount, rootBlock.Header.Height, rootBlock.ID()) + s.T().Logf("Generating %d blocks, root block: %d %s", blockCount, s.rootBlock.Header.Height, s.rootBlock.ID()) for i := 0; i < blockCount; i++ { block := unittest.BlockWithParentFixture(parent) @@ -158,13 +167,13 @@ func (s *BackendExecutionDataSuite) SetupTest() { s.registerID = unittest.RegisterIDFixture() - s.eventsIndex = backend.NewEventsIndex(s.events) + s.eventsIndex = index.NewEventsIndex(s.events) s.registersAsync = execution.NewRegistersAsyncStore() s.registers = storagemock.NewRegisterIndex(s.T()) err = s.registersAsync.Initialize(s.registers) require.NoError(s.T(), err) - s.registers.On("LatestHeight").Return(rootBlock.Header.Height).Maybe() - s.registers.On("FirstHeight").Return(rootBlock.Header.Height).Maybe() + s.registers.On("LatestHeight").Return(s.rootBlock.Header.Height).Maybe() + s.registers.On("FirstHeight").Return(s.rootBlock.Header.Height).Maybe() s.registers.On("Get", mock.AnythingOfType("RegisterID"), mock.AnythingOfType("uint64")).Return( func(id flow.RegisterID, height uint64) (flow.RegisterValue, error) { if id == s.registerID { @@ -219,13 +228,33 @@ func (s *BackendExecutionDataSuite) SetupTest() { s.eds, s.execDataCache, s.broadcaster, - rootBlock.Header.Height, - rootBlock.Header.Height, // initialize with no downloaded data s.registersAsync, s.eventsIndex, false, + s.executionDataTracker, ) require.NoError(s.T(), err) + + // create real execution data tracker to use GetStartHeight from it, instead of mocking + s.executionDataTrackerReal = subscription.NewExecutionDataTracker( + logger, + s.state, + s.rootBlock.Header.Height, + s.headers, + s.broadcaster, + s.rootBlock.Header.Height, + s.eventsIndex, + false, + ) + + s.executionDataTracker.On( + "GetStartHeight", + mock.Anything, + mock.Anything, + mock.Anything, + ).Return(func(ctx context.Context, startBlockID flow.Identifier, startHeight uint64) (uint64, error) { + return s.executionDataTrackerReal.GetStartHeight(ctx, startBlockID, startHeight) + }, nil).Maybe() } // generateMockEvents generates a set of mock events for a block split into multiple tx with @@ -266,7 +295,8 @@ func (s *BackendExecutionDataSuite) TestGetExecutionDataByBlockID() { execData := s.execDataMap[block.ID()] // notify backend block is available - s.backend.setHighestHeight(block.Header.Height) + s.executionDataTracker.On("GetHighestHeight"). + Return(block.Header.Height) var err error s.Run("happy path TestGetExecutionDataByBlockID success", func() { @@ -321,12 +351,12 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionData() { name: "happy path - start from root block by height", highestBackfill: len(s.blocks) - 1, // backfill all blocks startBlockID: flow.ZeroID, - startHeight: s.backend.rootBlockHeight, // start from root block + startHeight: s.rootBlock.Header.Height, // start from root block }, { name: "happy path - start from root block by id", - highestBackfill: len(s.blocks) - 1, // backfill all blocks - startBlockID: s.backend.rootBlockID, // start from root block + highestBackfill: len(s.blocks) - 1, // backfill all blocks + startBlockID: s.rootBlock.Header.ID(), // start from root block startHeight: 0, }, } @@ -342,21 +372,24 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionData() { // this simulates a subscription on a past block for i := 0; i <= test.highestBackfill; i++ { s.T().Logf("backfilling block %d", i) - s.backend.setHighestHeight(s.blocks[i].Header.Height) + s.executionDataTracker.On("GetHighestHeight"). + Return(s.blocks[i].Header.Height) } subCtx, subCancel := context.WithCancel(ctx) sub := s.backend.SubscribeExecutionData(subCtx, test.startBlockID, test.startHeight) - // loop over all of the blocks + // loop over of the all blocks for i, b := range s.blocks { execData := s.execDataMap[b.ID()] - s.T().Logf("checking block %d %v", i, b.ID()) + s.T().Logf("checking block %d %v %v", i, b.Header.Height, b.ID()) // simulate new exec data received. // exec data for all blocks with index <= highestBackfill were already received if i > test.highestBackfill { - s.backend.setHighestHeight(b.Header.Height) + s.executionDataTracker.On("GetHighestHeight").Unset() + s.executionDataTracker.On("GetHighestHeight"). + Return(b.Header.Height) s.broadcaster.Publish() } @@ -408,7 +441,7 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionDataHandlesErrors() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() - sub := s.backend.SubscribeExecutionData(subCtx, flow.ZeroID, s.backend.rootBlockHeight-1) + sub := s.backend.SubscribeExecutionData(subCtx, flow.ZeroID, s.rootBlock.Header.Height-1) assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err())) }) @@ -434,26 +467,26 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionDataHandlesErrors() { func (s *BackendExecutionDataSuite) TestGetRegisterValues() { s.Run("normal case", func() { - res, err := s.backend.GetRegisterValues(flow.RegisterIDs{s.registerID}, s.backend.rootBlockHeight) + res, err := s.backend.GetRegisterValues(flow.RegisterIDs{s.registerID}, s.rootBlock.Header.Height) require.NoError(s.T(), err) require.NotEmpty(s.T(), res) }) s.Run("returns error if block height is out of range", func() { - res, err := s.backend.GetRegisterValues(flow.RegisterIDs{s.registerID}, s.backend.rootBlockHeight+1) + res, err := s.backend.GetRegisterValues(flow.RegisterIDs{s.registerID}, s.rootBlock.Header.Height+1) require.Nil(s.T(), res) require.Equal(s.T(), codes.OutOfRange, status.Code(err)) }) s.Run("returns error if register path is not indexed", func() { falseID := flow.RegisterIDs{flow.RegisterID{Owner: "ha", Key: "ha"}} - res, err := s.backend.GetRegisterValues(falseID, s.backend.rootBlockHeight) + res, err := s.backend.GetRegisterValues(falseID, s.rootBlock.Header.Height) require.Nil(s.T(), res) require.Equal(s.T(), codes.NotFound, status.Code(err)) }) s.Run("returns error if too many registers are requested", func() { - res, err := s.backend.GetRegisterValues(make(flow.RegisterIDs, s.backend.registerRequestLimit+1), s.backend.rootBlockHeight) + res, err := s.backend.GetRegisterValues(make(flow.RegisterIDs, s.backend.registerRequestLimit+1), s.rootBlock.Header.Height) require.Nil(s.T(), res) require.Equal(s.T(), codes.InvalidArgument, status.Code(err)) }) diff --git a/engine/access/state_stream/backend/engine.go b/engine/access/state_stream/backend/engine.go index f6a9557862e..fb9196c6703 100644 --- a/engine/access/state_stream/backend/engine.go +++ b/engine/access/state_stream/backend/engine.go @@ -5,15 +5,12 @@ import ( "github.com/onflow/flow/protobuf/go/flow/executiondata" - "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/logging" ) // Engine exposes the server with the state stream API. @@ -27,9 +24,8 @@ type Engine struct { chain flow.Chain handler *Handler - execDataBroadcaster *engine.Broadcaster - execDataCache *cache.ExecutionDataCache - headers storage.Headers + execDataCache *cache.ExecutionDataCache + headers storage.Headers } // NewEng returns a new ingress server. @@ -41,19 +37,17 @@ func NewEng( chainID flow.ChainID, server *grpcserver.GrpcServer, backend *StateStreamBackend, - broadcaster *engine.Broadcaster, ) (*Engine, error) { logger := log.With().Str("engine", "state_stream_rpc").Logger() e := &Engine{ - log: logger, - backend: backend, - headers: headers, - chain: chainID.Chain(), - config: config, - handler: NewHandler(backend, chainID.Chain(), config), - execDataBroadcaster: broadcaster, - execDataCache: execDataCache, + log: logger, + backend: backend, + headers: headers, + chain: chainID.Chain(), + config: config, + handler: NewHandler(backend, chainID.Chain(), config), + execDataCache: execDataCache, } e.ComponentManager = component.NewComponentManagerBuilder(). @@ -67,30 +61,3 @@ func NewEng( return e, nil } - -// OnExecutionData is called to notify the engine when a new execution data is received. -// The caller must guarantee that execution data is locally available for all blocks with -// heights between the initialBlockHeight provided during startup and the block height of -// the execution data provided. -func (e *Engine) OnExecutionData(executionData *execution_data.BlockExecutionDataEntity) { - lg := e.log.With().Hex("block_id", logging.ID(executionData.BlockID)).Logger() - - lg.Trace().Msg("received execution data") - - header, err := e.headers.ByBlockID(executionData.BlockID) - if err != nil { - // if the execution data is available, the block must be locally finalized - lg.Fatal().Err(err).Msg("failed to get header for execution data") - return - } - - if ok := e.backend.setHighestHeight(header.Height); !ok { - // this means that the height was lower than the current highest height - // OnExecutionData is guaranteed by the requester to be called in order, but may be called - // multiple times for the same block. - lg.Debug().Msg("execution data for block already received") - return - } - - e.execDataBroadcaster.Publish() -} diff --git a/engine/access/state_stream/backend/handler.go b/engine/access/state_stream/backend/handler.go index 6ed22589562..2e8a61c8ac0 100644 --- a/engine/access/state_stream/backend/handler.go +++ b/engine/access/state_stream/backend/handler.go @@ -2,7 +2,6 @@ package backend import ( "context" - "sync/atomic" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -11,29 +10,28 @@ import ( "github.com/onflow/flow/protobuf/go/flow/executiondata" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" ) type Handler struct { + subscription.StreamingData + api state_stream.API chain flow.Chain - eventFilterConfig state_stream.EventFilterConfig - - maxStreams int32 - streamCount atomic.Int32 + eventFilterConfig state_stream.EventFilterConfig defaultHeartbeatInterval uint64 } func NewHandler(api state_stream.API, chain flow.Chain, config Config) *Handler { h := &Handler{ + StreamingData: subscription.NewStreamingData(config.MaxGlobalStreams), api: api, chain: chain, eventFilterConfig: config.EventFilterConfig, - maxStreams: int32(config.MaxGlobalStreams), - streamCount: atomic.Int32{}, defaultHeartbeatInterval: config.HeartbeatInterval, } return h @@ -65,11 +63,11 @@ func (h *Handler) GetExecutionDataByBlockID(ctx context.Context, request *execut func (h *Handler) SubscribeExecutionData(request *executiondata.SubscribeExecutionDataRequest, stream executiondata.ExecutionDataAPI_SubscribeExecutionDataServer) error { // check if the maximum number of streams is reached - if h.streamCount.Load() >= h.maxStreams { + if h.StreamCount.Load() >= h.MaxStreams { return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") } - h.streamCount.Add(1) - defer h.streamCount.Add(-1) + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) startBlockID := flow.ZeroID if request.GetStartBlockId() != nil { @@ -118,11 +116,11 @@ func (h *Handler) SubscribeExecutionData(request *executiondata.SubscribeExecuti func (h *Handler) SubscribeEvents(request *executiondata.SubscribeEventsRequest, stream executiondata.ExecutionDataAPI_SubscribeEventsServer) error { // check if the maximum number of streams is reached - if h.streamCount.Load() >= h.maxStreams { + if h.StreamCount.Load() >= h.MaxStreams { return status.Errorf(codes.ResourceExhausted, "maximum number of streams reached") } - h.streamCount.Add(1) - defer h.streamCount.Add(-1) + h.StreamCount.Add(1) + defer h.StreamCount.Add(-1) startBlockID := flow.ZeroID if request.GetStartBlockId() != nil { diff --git a/engine/access/state_stream/backend/handler_test.go b/engine/access/state_stream/backend/handler_test.go index 3cf9d656f8a..6670ce56c09 100644 --- a/engine/access/state_stream/backend/handler_test.go +++ b/engine/access/state_stream/backend/handler_test.go @@ -24,6 +24,7 @@ import ( "github.com/onflow/flow-go/engine/access/state_stream" ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/storage" @@ -79,7 +80,9 @@ func (s *HandlerTestSuite) TestHeartbeatResponse() { } // notify backend block is available - s.backend.setHighestHeight(s.blocks[len(s.blocks)-1].Header.Height) + s.executionDataTracker.On( + "GetHighestHeight"). + Return(s.blocks[len(s.blocks)-1].Header.Height) s.Run("All events filter", func() { // create empty event filter @@ -277,7 +280,7 @@ func TestExecutionDataStream(t *testing.T) { request *executiondata.SubscribeExecutionDataRequest, response *ExecutionDataResponse, ) { - sub := NewSubscription(1) + sub := subscription.NewSubscription(1) api.On("SubscribeExecutionData", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) @@ -403,7 +406,7 @@ func TestEventStream(t *testing.T) { request *executiondata.SubscribeEventsRequest, response *EventsResponse, ) { - sub := NewSubscription(1) + sub := subscription.NewSubscription(1) api.On("SubscribeEvents", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) @@ -615,10 +618,10 @@ func generateEvents(t *testing.T, n int) ([]flow.Event, []flow.Event) { func makeConfig(maxGlobalStreams uint32) Config { return Config{ EventFilterConfig: state_stream.DefaultEventFilterConfig, - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, + ClientSendTimeout: subscription.DefaultSendTimeout, + ClientSendBufferSize: subscription.DefaultSendBufferSize, MaxGlobalStreams: maxGlobalStreams, - HeartbeatInterval: state_stream.DefaultHeartbeatInterval, + HeartbeatInterval: subscription.DefaultHeartbeatInterval, } } diff --git a/engine/access/state_stream/mock/api.go b/engine/access/state_stream/mock/api.go index 99203a9f487..ed96c8c90e8 100644 --- a/engine/access/state_stream/mock/api.go +++ b/engine/access/state_stream/mock/api.go @@ -11,6 +11,8 @@ import ( mock "github.com/stretchr/testify/mock" state_stream "github.com/onflow/flow-go/engine/access/state_stream" + + subscription "github.com/onflow/flow-go/engine/access/subscription" ) // API is an autogenerated mock type for the API type @@ -71,15 +73,15 @@ func (_m *API) GetRegisterValues(registerIDs flow.RegisterIDs, height uint64) ([ } // SubscribeEvents provides a mock function with given fields: ctx, startBlockID, startHeight, filter -func (_m *API) SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter state_stream.EventFilter) state_stream.Subscription { +func (_m *API) SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter state_stream.EventFilter) subscription.Subscription { ret := _m.Called(ctx, startBlockID, startHeight, filter) - var r0 state_stream.Subscription - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64, state_stream.EventFilter) state_stream.Subscription); ok { + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64, state_stream.EventFilter) subscription.Subscription); ok { r0 = rf(ctx, startBlockID, startHeight, filter) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(state_stream.Subscription) + r0 = ret.Get(0).(subscription.Subscription) } } @@ -87,15 +89,15 @@ func (_m *API) SubscribeEvents(ctx context.Context, startBlockID flow.Identifier } // SubscribeExecutionData provides a mock function with given fields: ctx, startBlockID, startBlockHeight -func (_m *API) SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startBlockHeight uint64) state_stream.Subscription { +func (_m *API) SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startBlockHeight uint64) subscription.Subscription { ret := _m.Called(ctx, startBlockID, startBlockHeight) - var r0 state_stream.Subscription - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64) state_stream.Subscription); ok { + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64) subscription.Subscription); ok { r0 = rf(ctx, startBlockID, startBlockHeight) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(state_stream.Subscription) + r0 = ret.Get(0).(subscription.Subscription) } } diff --git a/engine/access/state_stream/state_stream.go b/engine/access/state_stream/state_stream.go index 2d2cca1bbbf..51ccc0d91bc 100644 --- a/engine/access/state_stream/state_stream.go +++ b/engine/access/state_stream/state_stream.go @@ -2,35 +2,13 @@ package state_stream import ( "context" - "time" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/executiondatasync/execution_data" ) const ( - // DefaultSendBufferSize is the default buffer size for the subscription's send channel. - // The size is chosen to balance memory overhead from each subscription with performance when - // streaming existing data. - DefaultSendBufferSize = 10 - - // DefaultMaxGlobalStreams defines the default max number of streams that can be open at the same time. - DefaultMaxGlobalStreams = 1000 - - // DefaultCacheSize defines the default max number of objects for the execution data cache. - DefaultCacheSize = 100 - - // DefaultSendTimeout is the default timeout for sending a message to the client. After the timeout - // expires, the connection is closed. - DefaultSendTimeout = 30 * time.Second - - // DefaultResponseLimit is default max responses per second allowed on a stream. After exceeding - // the limit, the stream is paused until more capacity is available. - DefaultResponseLimit = float64(0) - - // DefaultHeartbeatInterval specifies the block interval at which heartbeat messages should be sent. - DefaultHeartbeatInterval = 1 - // DefaultRegisterIDsRequestLimit defines the default limit of register IDs for a single request to the get register endpoint DefaultRegisterIDsRequestLimit = 100 ) @@ -40,31 +18,9 @@ type API interface { // GetExecutionDataByBlockID retrieves execution data for a specific block by its block ID. GetExecutionDataByBlockID(ctx context.Context, blockID flow.Identifier) (*execution_data.BlockExecutionData, error) // SubscribeExecutionData subscribes to execution data starting from a specific block ID and block height. - SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startBlockHeight uint64) Subscription + SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startBlockHeight uint64) subscription.Subscription // SubscribeEvents subscribes to events starting from a specific block ID and block height, with an optional event filter. - SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter EventFilter) Subscription + SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter EventFilter) subscription.Subscription // GetRegisterValues returns register values for a set of register IDs at the provided block height. GetRegisterValues(registerIDs flow.RegisterIDs, height uint64) ([]flow.RegisterValue, error) } - -// Subscription represents a streaming request, and handles the communication between the grpc handler -// and the backend implementation. -type Subscription interface { - // ID returns the unique identifier for this subscription used for logging - ID() string - - // Channel returns the channel from which subscription data can be read - Channel() <-chan interface{} - - // Err returns the error that caused the subscription to fail - Err() error -} - -// Streamable represents a subscription that can be streamed. -type Streamable interface { - ID() string - Close() - Fail(error) - Send(context.Context, interface{}, time.Duration) error - Next(context.Context) (interface{}, error) -} diff --git a/engine/access/subscription/base_tracker.go b/engine/access/subscription/base_tracker.go new file mode 100644 index 00000000000..e5905955e77 --- /dev/null +++ b/engine/access/subscription/base_tracker.go @@ -0,0 +1,193 @@ +package subscription + +import ( + "context" + "fmt" + "sync/atomic" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/common/rpc" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +// GetStartHeightFunc is a function type for getting the start height. +type GetStartHeightFunc func(context.Context, flow.Identifier, uint64) (uint64, error) + +// StreamingData represents common streaming data configuration for access and state_stream handlers. +type StreamingData struct { + MaxStreams int32 + StreamCount atomic.Int32 +} + +func NewStreamingData(maxStreams uint32) StreamingData { + return StreamingData{ + MaxStreams: int32(maxStreams), + StreamCount: atomic.Int32{}, + } +} + +// BaseTracker is an interface for a tracker that provides base GetStartHeight method related to both blocks and execution data tracking. +type BaseTracker interface { + // GetStartHeightFromBlockID returns the start height based on the provided starting block ID. + // If the start block is the root block, skip it and begin from the next block. + // + // Parameters: + // - startBlockID: The identifier of the starting block. + // + // Returns: + // - uint64: The start height associated with the provided block ID. + // - error: An error indicating any issues with retrieving the start height. + // + // Expected errors during normal operation: + // - codes.NotFound - if the block was not found in storage + // - codes.Internal - for any other error + GetStartHeightFromBlockID(flow.Identifier) (uint64, error) + // GetStartHeightFromHeight returns the start height based on the provided starting block height. + // If the start block is the root block, skip it and begin from the next block. + // + // Parameters: + // - startHeight: The height of the starting block. + // + // Returns: + // - uint64: The start height associated with the provided block height. + // - error: An error indicating any issues with retrieving the start height. + // + // Expected errors during normal operation: + // - codes.InvalidArgument - if the start height is less than the root block height. + // - codes.NotFound - if the header was not found in storage. + GetStartHeightFromHeight(uint64) (uint64, error) + // GetStartHeightFromLatest returns the start height based on the latest sealed block. + // If the start block is the root block, skip it and begin from the next block. + // + // Parameters: + // - ctx: Context for the operation. + // + // No errors are expected during normal operation. + GetStartHeightFromLatest(context.Context) (uint64, error) +} + +var _ BaseTracker = (*BaseTrackerImpl)(nil) + +// BaseTrackerImpl is an implementation of the BaseTracker interface. +type BaseTrackerImpl struct { + rootBlockHeight uint64 + state protocol.State + headers storage.Headers +} + +// NewBaseTrackerImpl creates a new instance of BaseTrackerImpl. +// +// Parameters: +// - rootBlockHeight: The root block height, which serves as the baseline for calculating the start height. +// - state: The protocol state used for retrieving block information. +// - headers: The storage headers for accessing block headers. +// +// Returns: +// - *BaseTrackerImpl: A new instance of BaseTrackerImpl. +func NewBaseTrackerImpl( + rootBlockHeight uint64, + state protocol.State, + headers storage.Headers, +) *BaseTrackerImpl { + return &BaseTrackerImpl{ + rootBlockHeight: rootBlockHeight, + state: state, + headers: headers, + } +} + +// GetStartHeightFromBlockID returns the start height based on the provided starting block ID. +// If the start block is the root block, skip it and begin from the next block. +// +// Parameters: +// - startBlockID: The identifier of the starting block. +// +// Returns: +// - uint64: The start height associated with the provided block ID. +// - error: An error indicating any issues with retrieving the start height. +// +// Expected errors during normal operation: +// - codes.NotFound - if the block was not found in storage +// - codes.Internal - for any other error +func (b *BaseTrackerImpl) GetStartHeightFromBlockID(startBlockID flow.Identifier) (uint64, error) { + header, err := b.headers.ByBlockID(startBlockID) + if err != nil { + return 0, rpc.ConvertStorageError(fmt.Errorf("could not get header for block %v: %w", startBlockID, err)) + } + + // ensure that the resolved start height is available + return b.checkStartHeight(header.Height), nil +} + +// GetStartHeightFromHeight returns the start height based on the provided starting block height. +// If the start block is the root block, skip it and begin from the next block. +// +// Parameters: +// - startHeight: The height of the starting block. +// +// Returns: +// - uint64: The start height associated with the provided block height. +// - error: An error indicating any issues with retrieving the start height. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if the start height is less than the root block height. +// - codes.NotFound - if the header was not found in storage. +func (b *BaseTrackerImpl) GetStartHeightFromHeight(startHeight uint64) (uint64, error) { + if startHeight < b.rootBlockHeight { + return 0, status.Errorf(codes.InvalidArgument, "start height must be greater than or equal to the root height %d", b.rootBlockHeight) + } + + header, err := b.headers.ByHeight(startHeight) + if err != nil { + return 0, rpc.ConvertStorageError(fmt.Errorf("could not get header for height %d: %w", startHeight, err)) + } + + // ensure that the resolved start height is available + return b.checkStartHeight(header.Height), nil +} + +// GetStartHeightFromLatest returns the start height based on the latest sealed block. +// If the start block is the root block, skip it and begin from the next block. +// +// Parameters: +// - ctx: Context for the operation. +// +// No errors are expected during normal operation. +func (b *BaseTrackerImpl) GetStartHeightFromLatest(ctx context.Context) (uint64, error) { + // if no start block was provided, use the latest sealed block + header, err := b.state.Sealed().Head() + if err != nil { + // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, + // we should halt processing requests + err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) + irrecoverable.Throw(ctx, err) + return 0, err + } + + return b.checkStartHeight(header.Height), nil +} + +// checkStartHeight validates the provided start height and adjusts it if necessary. +// If the start block is the root block, skip it and begin from the next block. +// +// Parameters: +// - height: The start height to be checked. +// +// Returns: +// - uint64: The adjusted start height. +// +// No errors are expected during normal operation. +func (b *BaseTrackerImpl) checkStartHeight(height uint64) uint64 { + // if the start block is the root block, skip it and begin from the next block. + if height == b.rootBlockHeight { + height = b.rootBlockHeight + 1 + } + + return height +} diff --git a/engine/access/subscription/block_tracker.go b/engine/access/subscription/block_tracker.go new file mode 100644 index 00000000000..51be3726fbd --- /dev/null +++ b/engine/access/subscription/block_tracker.go @@ -0,0 +1,123 @@ +package subscription + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +// BlockTracker is an interface for tracking blocks and handling block-related operations. +type BlockTracker interface { + BaseTracker + // GetHighestHeight returns the highest height based on the specified block status which could be only BlockStatusSealed + // or BlockStatusFinalized. + // No errors are expected during normal operation. + GetHighestHeight(flow.BlockStatus) (uint64, error) + // ProcessOnFinalizedBlock drives the subscription logic when a block is finalized. + // The input to this callback is treated as trusted. This method should be executed on + // `OnFinalizedBlock` notifications from the node-internal consensus instance. + // No errors are expected during normal operation. + ProcessOnFinalizedBlock() error +} + +var _ BlockTracker = (*BlockTrackerImpl)(nil) + +// BlockTrackerImpl is an implementation of the BlockTracker interface. +type BlockTrackerImpl struct { + BaseTracker + state protocol.State + broadcaster *engine.Broadcaster + + // finalizedHighestHeight contains the highest consecutive block height for which we have received a new notification. + finalizedHighestHeight counters.StrictMonotonousCounter + // sealedHighestHeight contains the highest consecutive block height for which we have received a new notification. + sealedHighestHeight counters.StrictMonotonousCounter +} + +// NewBlockTracker creates a new BlockTrackerImpl instance. +// +// Parameters: +// - state: The protocol state used for retrieving block information. +// - rootHeight: The root block height, serving as the baseline for calculating the start height. +// - headers: The storage headers for accessing block headers. +// - broadcaster: The engine broadcaster for publishing notifications. +// +// No errors are expected during normal operation. +func NewBlockTracker( + state protocol.State, + rootHeight uint64, + headers storage.Headers, + broadcaster *engine.Broadcaster, +) (*BlockTrackerImpl, error) { + lastFinalized, err := state.Final().Head() + if err != nil { + // this header MUST exist in the db, otherwise the node likely has inconsistent state. + return nil, irrecoverable.NewExceptionf("could not retrieve last finalized block: %w", err) + } + + lastSealed, err := state.Sealed().Head() + if err != nil { + // this header MUST exist in the db, otherwise the node likely has inconsistent state. + return nil, irrecoverable.NewExceptionf("could not retrieve last sealed block: %w", err) + } + + return &BlockTrackerImpl{ + BaseTracker: NewBaseTrackerImpl(rootHeight, state, headers), + state: state, + finalizedHighestHeight: counters.NewMonotonousCounter(lastFinalized.Height), + sealedHighestHeight: counters.NewMonotonousCounter(lastSealed.Height), + broadcaster: broadcaster, + }, nil +} + +// GetHighestHeight returns the highest height based on the specified block status. +// +// Parameters: +// - blockStatus: The status of the block. It is expected that blockStatus has already been handled for invalid flow.BlockStatusUnknown. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if block status is flow.BlockStatusUnknown. +func (b *BlockTrackerImpl) GetHighestHeight(blockStatus flow.BlockStatus) (uint64, error) { + switch blockStatus { + case flow.BlockStatusFinalized: + return b.finalizedHighestHeight.Value(), nil + case flow.BlockStatusSealed: + return b.sealedHighestHeight.Value(), nil + } + return 0, status.Errorf(codes.InvalidArgument, "invalid block status: %s", blockStatus) +} + +// ProcessOnFinalizedBlock drives the subscription logic when a block is finalized. +// The input to this callback is treated as trusted. This method should be executed on +// `OnFinalizedBlock` notifications from the node-internal consensus instance. +// No errors are expected during normal operation. Any errors encountered should be +// treated as an exception. +func (b *BlockTrackerImpl) ProcessOnFinalizedBlock() error { + // get the finalized header from state + finalizedHeader, err := b.state.Final().Head() + if err != nil { + return irrecoverable.NewExceptionf("unable to get latest finalized header: %w", err) + } + + if !b.finalizedHighestHeight.Set(finalizedHeader.Height) { + return nil + } + + // get the latest seal header from state + sealedHeader, err := b.state.Sealed().Head() + if err != nil { + return irrecoverable.NewExceptionf("unable to get latest sealed header: %w", err) + } + + _ = b.sealedHighestHeight.Set(sealedHeader.Height) + // always publish since there is also a new finalized block. + b.broadcaster.Publish() + + return nil +} diff --git a/engine/access/subscription/execution_data_tracker.go b/engine/access/subscription/execution_data_tracker.go new file mode 100644 index 00000000000..76efbf07513 --- /dev/null +++ b/engine/access/subscription/execution_data_tracker.go @@ -0,0 +1,248 @@ +package subscription + +import ( + "context" + + "github.com/rs/zerolog" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/common/rpc" + "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/state_synchronization" + "github.com/onflow/flow-go/module/state_synchronization/indexer" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/logging" +) + +// ExecutionDataTracker is an interface for tracking the highest consecutive block height for which we have received a +// new Execution Data notification +type ExecutionDataTracker interface { + BaseTracker + // GetStartHeight returns the start height to use when searching. + // Only one of startBlockID and startHeight may be set. Otherwise, an InvalidArgument error is returned. + // If a block is provided and does not exist, a NotFound error is returned. + // If neither startBlockID nor startHeight is provided, the latest sealed block is used. + // If the start block is the root block, skip it and begin from the next block. + // + // Parameters: + // - ctx: Context for the operation. + // - startBlockID: The identifier of the starting block. If provided, startHeight should be 0. + // - startHeight: The height of the starting block. If provided, startBlockID should be flow.ZeroID. + // + // Returns: + // - uint64: The start height for searching. + // - error: An error indicating the result of the operation, if any. + // + // Expected errors during normal operation: + // - codes.InvalidArgument - if both startBlockID and startHeight are provided, if the start height is less than the root block height, + // if the start height is out of bounds based on indexed heights (when index is used). + // - codes.NotFound - if a block is provided and does not exist. + // - codes.Internal - if there is an internal error. + GetStartHeight(context.Context, flow.Identifier, uint64) (uint64, error) + // GetHighestHeight returns the highest height that we have consecutive execution data for. + GetHighestHeight() uint64 + // OnExecutionData is used to notify the tracker when a new execution data is received. + OnExecutionData(*execution_data.BlockExecutionDataEntity) +} + +var _ ExecutionDataTracker = (*ExecutionDataTrackerImpl)(nil) + +// ExecutionDataTrackerImpl is an implementation of the ExecutionDataTracker interface. +type ExecutionDataTrackerImpl struct { + BaseTracker + log zerolog.Logger + headers storage.Headers + broadcaster *engine.Broadcaster + indexReporter state_synchronization.IndexReporter + useIndex bool + + // highestHeight contains the highest consecutive block height that we have consecutive execution data for + highestHeight counters.StrictMonotonousCounter +} + +// NewExecutionDataTracker creates a new ExecutionDataTrackerImpl instance. +// +// Parameters: +// - log: The logger to use for logging. +// - state: The protocol state used for retrieving block information. +// - rootHeight: The root block height, serving as the baseline for calculating the start height. +// - headers: The storage headers for accessing block headers. +// - broadcaster: The engine broadcaster for publishing notifications. +// - highestAvailableFinalizedHeight: The highest available finalized block height. +// - indexReporter: The index reporter for checking indexed block heights. +// - useIndex: A flag indicating whether to use indexed block heights for validation. +// +// Returns: +// - *ExecutionDataTrackerImpl: A new instance of ExecutionDataTrackerImpl. +func NewExecutionDataTracker( + log zerolog.Logger, + state protocol.State, + rootHeight uint64, + headers storage.Headers, + broadcaster *engine.Broadcaster, + highestAvailableFinalizedHeight uint64, + indexReporter state_synchronization.IndexReporter, + useIndex bool, +) *ExecutionDataTrackerImpl { + return &ExecutionDataTrackerImpl{ + BaseTracker: NewBaseTrackerImpl(rootHeight, state, headers), + log: log, + headers: headers, + broadcaster: broadcaster, + highestHeight: counters.NewMonotonousCounter(highestAvailableFinalizedHeight), + indexReporter: indexReporter, + useIndex: useIndex, + } +} + +// GetStartHeight returns the start height to use when searching. +// Only one of startBlockID and startHeight may be set. Otherwise, an InvalidArgument error is returned. +// If a block is provided and does not exist, a NotFound error is returned. +// If neither startBlockID nor startHeight is provided, the latest sealed block is used. +// If the start block is the root block, skip it and begin from the next block. +// +// Parameters: +// - ctx: Context for the operation. +// - startBlockID: The identifier of the starting block. If provided, startHeight should be 0. +// - startHeight: The height of the starting block. If provided, startBlockID should be flow.ZeroID. +// +// Returns: +// - uint64: The start height for searching. +// - error: An error indicating the result of the operation, if any. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if both startBlockID and startHeight are provided, if the start height is less than the root block height, +// if the start height is out of bounds based on indexed heights (when index is used). +// - codes.NotFound - if a block is provided and does not exist. +// - codes.Internal - if there is an internal error. +func (e *ExecutionDataTrackerImpl) GetStartHeight(ctx context.Context, startBlockID flow.Identifier, startHeight uint64) (uint64, error) { + var height uint64 = 0 + var err error + + if startBlockID != flow.ZeroID && startHeight > 0 { + return 0, status.Errorf(codes.InvalidArgument, "only one of start block ID and start height may be provided") + } + + // get the start height based on the provided starting block ID + if startBlockID != flow.ZeroID { + height, err = e.GetStartHeightFromBlockID(startBlockID) + } + + // get start height based on the provided starting block height + if startHeight > 0 { + height, err = e.GetStartHeightFromHeight(startHeight) + } + + // get start height based latest sealed block when neither startBlockID nor startHeight is provided + if startBlockID == flow.ZeroID && startHeight == 0 { + height, err = e.GetStartHeightFromLatest(ctx) + } + + if err != nil { + return 0, err + } + + // ensure that the resolved start height is available + return e.checkStartHeight(height) +} + +// GetHighestHeight returns the highest height that we have consecutive execution data for. +func (e *ExecutionDataTrackerImpl) GetHighestHeight() uint64 { + return e.highestHeight.Value() +} + +// OnExecutionData is used to notify the tracker when a new execution data is received. +func (e *ExecutionDataTrackerImpl) OnExecutionData(executionData *execution_data.BlockExecutionDataEntity) { + log := e.log.With().Hex("block_id", logging.ID(executionData.BlockID)).Logger() + + log.Trace().Msg("received execution data") + + header, err := e.headers.ByBlockID(executionData.BlockID) + if err != nil { + // if the execution data is available, the block must be locally finalized + log.Fatal().Err(err).Msg("failed to notify of new execution data") + return + } + + // sets the highest height for which execution data is available. + _ = e.highestHeight.Set(header.Height) + + e.broadcaster.Publish() +} + +// checkStartHeight validates the provided start height and adjusts it if necessary based on the tracker's configuration. +// +// Parameters: +// - height: The start height to be checked. +// +// Returns: +// - uint64: The adjusted start height, if validation passes. +// - error: An error indicating any issues with the provided start height. +// +// Validation Steps: +// 1. If index usage is disabled, return the original height without further checks. +// 2. Retrieve the lowest and highest indexed block heights. +// 3. Check if the provided height is within the bounds of indexed heights. +// - If below the lowest indexed height, return codes.InvalidArgument error. +// - If above the highest indexed height, return codes.InvalidArgument error. +// +// 4. If validation passes, return the adjusted start height. +// +// Expected errors during normal operation: +// - codes.InvalidArgument - if both startBlockID and startHeight are provided, if the start height is less than the +// root block height, if the start height is out of bounds based on indexed heights. +// - codes.FailedPrecondition - if the index reporter is not ready yet. +// - codes.Internal - for any other error during validation. +func (e *ExecutionDataTrackerImpl) checkStartHeight(height uint64) (uint64, error) { + if !e.useIndex { + return height, nil + } + + lowestHeight, highestHeight, err := e.getIndexedHeightBound() + if err != nil { + return 0, err + } + + if height < lowestHeight { + return 0, status.Errorf(codes.InvalidArgument, "start height %d is lower than lowest indexed height %d", height, lowestHeight) + } + + if height > highestHeight { + return 0, status.Errorf(codes.InvalidArgument, "start height %d is higher than highest indexed height %d", height, highestHeight) + } + + return height, nil +} + +// getIndexedHeightBound returns the lowest and highest indexed block heights +// Expected errors during normal operation: +// - codes.FailedPrecondition - if the index reporter is not ready yet. +// - codes.Internal - if there was any other error getting the heights. +func (e *ExecutionDataTrackerImpl) getIndexedHeightBound() (uint64, uint64, error) { + lowestHeight, err := e.indexReporter.LowestIndexedHeight() + if err != nil { + if errors.Is(err, storage.ErrHeightNotIndexed) || errors.Is(err, indexer.ErrIndexNotInitialized) { + // the index is not ready yet, but likely will be eventually + return 0, 0, status.Errorf(codes.FailedPrecondition, "failed to get lowest indexed height: %v", err) + } + return 0, 0, rpc.ConvertError(err, "failed to get lowest indexed height", codes.Internal) + } + + highestHeight, err := e.indexReporter.HighestIndexedHeight() + if err != nil { + if errors.Is(err, storage.ErrHeightNotIndexed) || errors.Is(err, indexer.ErrIndexNotInitialized) { + // the index is not ready yet, but likely will be eventually + return 0, 0, status.Errorf(codes.FailedPrecondition, "failed to get highest indexed height: %v", err) + } + return 0, 0, rpc.ConvertError(err, "failed to get highest indexed height", codes.Internal) + } + + return lowestHeight, highestHeight, nil +} diff --git a/engine/access/subscription/mock/block_tracker.go b/engine/access/subscription/mock/block_tracker.go new file mode 100644 index 00000000000..6143ace674a --- /dev/null +++ b/engine/access/subscription/mock/block_tracker.go @@ -0,0 +1,140 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// BlockTracker is an autogenerated mock type for the BlockTracker type +type BlockTracker struct { + mock.Mock +} + +// GetHighestHeight provides a mock function with given fields: _a0 +func (_m *BlockTracker) GetHighestHeight(_a0 flow.BlockStatus) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(flow.BlockStatus) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(flow.BlockStatus) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(flow.BlockStatus) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStartHeightFromBlockID provides a mock function with given fields: _a0 +func (_m *BlockTracker) GetStartHeightFromBlockID(_a0 flow.Identifier) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStartHeightFromHeight provides a mock function with given fields: _a0 +func (_m *BlockTracker) GetStartHeightFromHeight(_a0 uint64) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(uint64) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(uint64) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStartHeightFromLatest provides a mock function with given fields: _a0 +func (_m *BlockTracker) GetStartHeightFromLatest(_a0 context.Context) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProcessOnFinalizedBlock provides a mock function with given fields: +func (_m *BlockTracker) ProcessOnFinalizedBlock() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewBlockTracker interface { + mock.TestingT + Cleanup(func()) +} + +// NewBlockTracker creates a new instance of BlockTracker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBlockTracker(t mockConstructorTestingTNewBlockTracker) *BlockTracker { + mock := &BlockTracker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/subscription/mock/execution_data_tracker.go b/engine/access/subscription/mock/execution_data_tracker.go new file mode 100644 index 00000000000..fd477e15d9d --- /dev/null +++ b/engine/access/subscription/mock/execution_data_tracker.go @@ -0,0 +1,147 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + flow "github.com/onflow/flow-go/model/flow" + execution_data "github.com/onflow/flow-go/module/executiondatasync/execution_data" + + mock "github.com/stretchr/testify/mock" +) + +// ExecutionDataTracker is an autogenerated mock type for the ExecutionDataTracker type +type ExecutionDataTracker struct { + mock.Mock +} + +// GetHighestHeight provides a mock function with given fields: +func (_m *ExecutionDataTracker) GetHighestHeight() uint64 { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + return r0 +} + +// GetStartHeight provides a mock function with given fields: _a0, _a1, _a2 +func (_m *ExecutionDataTracker) GetStartHeight(_a0 context.Context, _a1 flow.Identifier, _a2 uint64) (uint64, error) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64) (uint64, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64) uint64); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint64) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStartHeightFromBlockID provides a mock function with given fields: _a0 +func (_m *ExecutionDataTracker) GetStartHeightFromBlockID(_a0 flow.Identifier) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStartHeightFromHeight provides a mock function with given fields: _a0 +func (_m *ExecutionDataTracker) GetStartHeightFromHeight(_a0 uint64) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(uint64) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(uint64) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStartHeightFromLatest provides a mock function with given fields: _a0 +func (_m *ExecutionDataTracker) GetStartHeightFromLatest(_a0 context.Context) (uint64, error) { + ret := _m.Called(_a0) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) uint64); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// OnExecutionData provides a mock function with given fields: _a0 +func (_m *ExecutionDataTracker) OnExecutionData(_a0 *execution_data.BlockExecutionDataEntity) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewExecutionDataTracker interface { + mock.TestingT + Cleanup(func()) +} + +// NewExecutionDataTracker creates a new instance of ExecutionDataTracker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewExecutionDataTracker(t mockConstructorTestingTNewExecutionDataTracker) *ExecutionDataTracker { + mock := &ExecutionDataTracker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/state_stream/backend/streamer.go b/engine/access/subscription/streamer.go similarity index 94% rename from engine/access/state_stream/backend/streamer.go rename to engine/access/subscription/streamer.go index df821154630..84fca744fc4 100644 --- a/engine/access/state_stream/backend/streamer.go +++ b/engine/access/subscription/streamer.go @@ -1,4 +1,4 @@ -package backend +package subscription import ( "context" @@ -10,26 +10,26 @@ import ( "golang.org/x/time/rate" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/storage" ) -// Streamer +// Streamer represents a streaming subscription that delivers data to clients. type Streamer struct { log zerolog.Logger - sub state_stream.Streamable + sub Streamable broadcaster *engine.Broadcaster sendTimeout time.Duration limiter *rate.Limiter } +// NewStreamer creates a new Streamer instance. func NewStreamer( log zerolog.Logger, broadcaster *engine.Broadcaster, sendTimeout time.Duration, limit float64, - sub state_stream.Streamable, + sub Streamable, ) *Streamer { var limiter *rate.Limiter if limit > 0 { diff --git a/engine/access/state_stream/backend/streamer_test.go b/engine/access/subscription/streamer_test.go similarity index 87% rename from engine/access/state_stream/backend/streamer_test.go rename to engine/access/subscription/streamer_test.go index 8226b5902f4..b3d46867c0d 100644 --- a/engine/access/state_stream/backend/streamer_test.go +++ b/engine/access/subscription/streamer_test.go @@ -1,4 +1,4 @@ -package backend_test +package subscription_test import ( "context" @@ -11,9 +11,8 @@ import ( "github.com/stretchr/testify/mock" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/access/state_stream" - "github.com/onflow/flow-go/engine/access/state_stream/backend" streammock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/utils/unittest" ) @@ -28,7 +27,7 @@ func TestStream(t *testing.T) { t.Parallel() ctx := context.Background() - timeout := state_stream.DefaultSendTimeout + timeout := subscription.DefaultSendTimeout sub := streammock.NewStreamable(t) sub.On("ID").Return(uuid.NewString()) @@ -40,7 +39,7 @@ func TestStream(t *testing.T) { tests = append(tests, testData{"", testErr}) broadcaster := engine.NewBroadcaster() - streamer := backend.NewStreamer(unittest.Logger(), broadcaster, timeout, state_stream.DefaultResponseLimit, sub) + streamer := subscription.NewStreamer(unittest.Logger(), broadcaster, timeout, subscription.DefaultResponseLimit, sub) for _, d := range tests { sub.On("Next", mock.Anything).Return(d.data, d.err).Once() @@ -65,7 +64,7 @@ func TestStreamRatelimited(t *testing.T) { t.Parallel() ctx := context.Background() - timeout := state_stream.DefaultSendTimeout + timeout := subscription.DefaultSendTimeout duration := 100 * time.Millisecond for _, limit := range []float64{0.2, 3, 20, 500} { @@ -74,7 +73,7 @@ func TestStreamRatelimited(t *testing.T) { sub.On("ID").Return(uuid.NewString()) broadcaster := engine.NewBroadcaster() - streamer := backend.NewStreamer(unittest.Logger(), broadcaster, timeout, limit, sub) + streamer := subscription.NewStreamer(unittest.Logger(), broadcaster, timeout, limit, sub) var nextCalls, sendCalls int sub.On("Next", mock.Anything).Return("data", nil).Run(func(args mock.Arguments) { @@ -116,7 +115,7 @@ func TestLongStreamRatelimited(t *testing.T) { unittest.SkipUnless(t, unittest.TEST_LONG_RUNNING, "skipping long stream rate limit test") ctx := context.Background() - timeout := state_stream.DefaultSendTimeout + timeout := subscription.DefaultSendTimeout limit := 5.0 duration := 30 * time.Second @@ -125,7 +124,7 @@ func TestLongStreamRatelimited(t *testing.T) { sub.On("ID").Return(uuid.NewString()) broadcaster := engine.NewBroadcaster() - streamer := backend.NewStreamer(unittest.Logger(), broadcaster, timeout, limit, sub) + streamer := subscription.NewStreamer(unittest.Logger(), broadcaster, timeout, limit, sub) var nextCalls, sendCalls int sub.On("Next", mock.Anything).Return("data", nil).Run(func(args mock.Arguments) { diff --git a/engine/access/state_stream/backend/subscription.go b/engine/access/subscription/subscription.go similarity index 60% rename from engine/access/state_stream/backend/subscription.go rename to engine/access/subscription/subscription.go index eb568d196db..3c5a12cee31 100644 --- a/engine/access/state_stream/backend/subscription.go +++ b/engine/access/subscription/subscription.go @@ -1,4 +1,4 @@ -package backend +package subscription import ( "context" @@ -8,8 +8,30 @@ import ( "github.com/google/uuid" "google.golang.org/grpc/status" +) + +const ( + // DefaultSendBufferSize is the default buffer size for the subscription's send channel. + // The size is chosen to balance memory overhead from each subscription with performance when + // streaming existing data. + DefaultSendBufferSize = 10 + + // DefaultMaxGlobalStreams defines the default max number of streams that can be open at the same time. + DefaultMaxGlobalStreams = 1000 + + // DefaultCacheSize defines the default max number of objects for the execution data cache. + DefaultCacheSize = 100 + + // DefaultSendTimeout is the default timeout for sending a message to the client. After the timeout + // expires, the connection is closed. + DefaultSendTimeout = 30 * time.Second + + // DefaultResponseLimit is default max responses per second allowed on a stream. After exceeding + // the limit, the stream is paused until more capacity is available. + DefaultResponseLimit = float64(0) - "github.com/onflow/flow-go/engine/access/state_stream" + // DefaultHeartbeatInterval specifies the block interval at which heartbeat messages should be sent. + DefaultHeartbeatInterval = 1 ) // GetDataByHeightFunc is a callback used by subscriptions to retrieve data for a given height. @@ -19,7 +41,38 @@ import ( // All other errors are considered exceptions type GetDataByHeightFunc func(ctx context.Context, height uint64) (interface{}, error) -var _ state_stream.Subscription = (*SubscriptionImpl)(nil) +// Subscription represents a streaming request, and handles the communication between the grpc handler +// and the backend implementation. +type Subscription interface { + // ID returns the unique identifier for this subscription used for logging + ID() string + + // Channel returns the channel from which subscription data can be read + Channel() <-chan interface{} + + // Err returns the error that caused the subscription to fail + Err() error +} + +// Streamable represents a subscription that can be streamed. +type Streamable interface { + // ID returns the subscription ID + // Note: this is not a cryptographic hash + ID() string + // Close is called when a subscription ends gracefully, and closes the subscription channel + Close() + // Fail registers an error and closes the subscription channel + Fail(error) + // Send sends a value to the subscription channel or returns an error + // Expected errors: + // - context.DeadlineExceeded if send timed out + // - context.Canceled if the client disconnected + Send(context.Context, interface{}, time.Duration) error + // Next returns the value for the next height from the subscription + Next(context.Context) (interface{}, error) +} + +var _ Subscription = (*SubscriptionImpl)(nil) type SubscriptionImpl struct { id string @@ -110,8 +163,8 @@ func NewFailedSubscription(err error, msg string) *SubscriptionImpl { return sub } -var _ state_stream.Subscription = (*HeightBasedSubscription)(nil) -var _ state_stream.Streamable = (*HeightBasedSubscription)(nil) +var _ Subscription = (*HeightBasedSubscription)(nil) +var _ Streamable = (*HeightBasedSubscription)(nil) // HeightBasedSubscription is a subscription that retrieves data sequentially by block height type HeightBasedSubscription struct { diff --git a/engine/access/state_stream/backend/subscription_test.go b/engine/access/subscription/subscription_test.go similarity index 90% rename from engine/access/state_stream/backend/subscription_test.go rename to engine/access/subscription/subscription_test.go index 2df54ecf570..a86422c17fd 100644 --- a/engine/access/state_stream/backend/subscription_test.go +++ b/engine/access/subscription/subscription_test.go @@ -1,4 +1,4 @@ -package backend_test +package subscription_test import ( "context" @@ -7,11 +7,10 @@ import ( "testing" "time" - "github.com/onflow/flow-go/engine/access/state_stream/backend" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/utils/unittest" ) @@ -21,7 +20,7 @@ func TestSubscription_SendReceive(t *testing.T) { ctx := context.Background() - sub := backend.NewSubscription(1) + sub := subscription.NewSubscription(1) assert.NotEmpty(t, sub.ID()) @@ -67,7 +66,7 @@ func TestSubscription_Failures(t *testing.T) { // make sure closing a subscription twice does not cause a panic t.Run("close only called once", func(t *testing.T) { - sub := backend.NewSubscription(1) + sub := subscription.NewSubscription(1) sub.Close() sub.Close() @@ -76,7 +75,7 @@ func TestSubscription_Failures(t *testing.T) { // make sure failing and closing the same subscription does not cause a panic t.Run("close only called once with fail", func(t *testing.T) { - sub := backend.NewSubscription(1) + sub := subscription.NewSubscription(1) sub.Fail(testErr) sub.Close() @@ -85,7 +84,7 @@ func TestSubscription_Failures(t *testing.T) { // make sure an error is returned when sending on a closed subscription t.Run("send after closed returns an error", func(t *testing.T) { - sub := backend.NewSubscription(1) + sub := subscription.NewSubscription(1) sub.Fail(testErr) err := sub.Send(context.Background(), "test", 10*time.Millisecond) @@ -118,7 +117,7 @@ func TestHeightBasedSubscription(t *testing.T) { } // search from [start, last], checking the correct data is returned - sub := backend.NewHeightBasedSubscription(1, start, getData) + sub := subscription.NewHeightBasedSubscription(1, start, getData) for i := start; i <= last; i++ { data, err := sub.Next(ctx) if err != nil { diff --git a/engine/access/subscription/util.go b/engine/access/subscription/util.go new file mode 100644 index 00000000000..593f3d78499 --- /dev/null +++ b/engine/access/subscription/util.go @@ -0,0 +1,39 @@ +package subscription + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/common/rpc" +) + +// HandleSubscription is a generic handler for subscriptions to a specific type. It continuously listens to the subscription channel, +// handles the received responses, and sends the processed information to the client via the provided stream using handleResponse. +// +// Parameters: +// - sub: The subscription. +// - handleResponse: The function responsible for handling the response of the subscribed type. +// +// Expected errors during normal operation: +// - codes.Internal: If the subscription encounters an error or gets an unexpected response. +func HandleSubscription[T any](sub Subscription, handleResponse func(resp T) error) error { + for { + v, ok := <-sub.Channel() + if !ok { + if sub.Err() != nil { + return rpc.ConvertError(sub.Err(), "stream encountered an error", codes.Internal) + } + return nil + } + + resp, ok := v.(T) + if !ok { + return status.Errorf(codes.Internal, "unexpected response type: %T", v) + } + + err := handleResponse(resp) + if err != nil { + return err + } + } +} diff --git a/engine/common/rpc/convert/blocks.go b/engine/common/rpc/convert/blocks.go index 62f1298911f..6e3588090ea 100644 --- a/engine/common/rpc/convert/blocks.go +++ b/engine/common/rpc/convert/blocks.go @@ -155,3 +155,16 @@ func PayloadFromMessage(m *entities.Block) (*flow.Payload, error) { ProtocolStateID: MessageToIdentifier(m.ProtocolStateId), }, nil } + +// MessageToBlockStatus converts a protobuf BlockStatus message to a flow.BlockStatus. +func MessageToBlockStatus(status entities.BlockStatus) flow.BlockStatus { + switch status { + case entities.BlockStatus_BLOCK_UNKNOWN: + return flow.BlockStatusUnknown + case entities.BlockStatus_BLOCK_FINALIZED: + return flow.BlockStatusFinalized + case entities.BlockStatus_BLOCK_SEALED: + return flow.BlockStatusSealed + } + return flow.BlockStatusUnknown +} diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index ceba4ed35bf..92245acf949 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -1404,7 +1404,8 @@ func TestSettingExecutionWeights(t *testing.T) { ).run( func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Use the maximum amount of computation so that the transaction still passes. - loops := uint64(997) + loops := uint64(996) + executionEffortNeededToCheckStorage := uint64(1) maxExecutionEffort := uint64(997) txBody := flow.NewTransactionBody(). SetScript([]byte(fmt.Sprintf(` @@ -1427,8 +1428,8 @@ func TestSettingExecutionWeights(t *testing.T) { snapshotTree = snapshotTree.Append(executionSnapshot) - // expected used is number of loops. - require.Equal(t, loops, output.ComputationUsed) + // expected computation used is number of loops + 1 (from the storage limit check). + require.Equal(t, loops+executionEffortNeededToCheckStorage, output.ComputationUsed) // increasing the number of loops should fail the transaction. loops = loops + 1 @@ -1451,8 +1452,8 @@ func TestSettingExecutionWeights(t *testing.T) { require.NoError(t, err) require.ErrorContains(t, output.Err, "computation exceeds limit (997)") - // computation used should the actual computation used. - require.Equal(t, loops, output.ComputationUsed) + // expected computation used is still number of loops + 1 (from the storage limit check). + require.Equal(t, loops+executionEffortNeededToCheckStorage, output.ComputationUsed) for _, event := range output.Events { // the fee deduction event should only contain the max gas worth of execution effort. @@ -1477,6 +1478,117 @@ func TestSettingExecutionWeights(t *testing.T) { unittest.EnsureEventsIndexSeq(t, output.Events, chain.ChainID()) }, )) + + t.Run("transaction with more accounts touched uses more computation", newVMTest().withBootstrapProcedureOptions( + fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation), + fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee), + fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), + fvm.WithTransactionFee(fvm.DefaultTransactionFees), + fvm.WithExecutionEffortWeights( + meter.ExecutionEffortWeights{ + common.ComputationKindStatement: 0, + // only count loops + // the storage check has a loop + common.ComputationKindLoop: 1 << meter.MeterExecutionInternalPrecisionBytes, + common.ComputationKindFunctionInvocation: 0, + }, + ), + ).withContextOptions( + fvm.WithAccountStorageLimit(true), + fvm.WithTransactionFeesEnabled(true), + fvm.WithMemoryLimit(math.MaxUint64), + ).run( + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { + // Create an account private key. + privateKeys, err := testutil.GenerateAccountPrivateKeys(5) + require.NoError(t, err) + + // Bootstrap a ledger, creating accounts with the provided + // private keys and the root account. + snapshotTree, accounts, err := testutil.CreateAccounts( + vm, + snapshotTree, + privateKeys, + chain) + require.NoError(t, err) + + sc := systemcontracts.SystemContractsForChain(chain.ChainID()) + + // create a transaction without loops so only the looping in the storage check is counted. + txBody := flow.NewTransactionBody(). + SetScript([]byte(fmt.Sprintf(` + import FungibleToken from 0x%s + import FlowToken from 0x%s + + transaction() { + let sentVault: @FungibleToken.Vault + + prepare(signer: AuthAccount) { + let vaultRef = signer.borrow<&FlowToken.Vault>(from: /storage/flowTokenVault) + ?? panic("Could not borrow reference to the owner's Vault!") + + self.sentVault <- vaultRef.withdraw(amount: 5.0) + } + + execute { + let recipient1 = getAccount(%s) + let recipient2 = getAccount(%s) + let recipient3 = getAccount(%s) + let recipient4 = getAccount(%s) + let recipient5 = getAccount(%s) + + let receiverRef1 = recipient1.getCapability(/public/flowTokenReceiver) + .borrow<&{FungibleToken.Receiver}>() + ?? panic("Could not borrow receiver reference to the recipient's Vault") + let receiverRef2 = recipient2.getCapability(/public/flowTokenReceiver) + .borrow<&{FungibleToken.Receiver}>() + ?? panic("Could not borrow receiver reference to the recipient's Vault") + let receiverRef3 = recipient3.getCapability(/public/flowTokenReceiver) + .borrow<&{FungibleToken.Receiver}>() + ?? panic("Could not borrow receiver reference to the recipient's Vault") + let receiverRef4 = recipient4.getCapability(/public/flowTokenReceiver) + .borrow<&{FungibleToken.Receiver}>() + ?? panic("Could not borrow receiver reference to the recipient's Vault") + let receiverRef5 = recipient5.getCapability(/public/flowTokenReceiver) + .borrow<&{FungibleToken.Receiver}>() + ?? panic("Could not borrow receiver reference to the recipient's Vault") + + receiverRef1.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) + receiverRef2.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) + receiverRef3.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) + receiverRef4.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) + receiverRef5.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) + + destroy self.sentVault + } + }`, + sc.FungibleToken.Address, + sc.FlowToken.Address, + accounts[0].HexWithPrefix(), + accounts[1].HexWithPrefix(), + accounts[2].HexWithPrefix(), + accounts[3].HexWithPrefix(), + accounts[4].HexWithPrefix(), + ))). + SetProposalKey(chain.ServiceAddress(), 0, 0). + AddAuthorizer(chain.ServiceAddress()). + SetPayer(chain.ServiceAddress()) + + err = testutil.SignTransactionAsServiceAccount(txBody, 0, chain) + require.NoError(t, err) + + _, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree) + require.NoError(t, err) + require.NoError(t, output.Err) + + // The storage check should loop once for each of the five accounts created + + // once for the service account + require.Equal(t, uint64(5+1), output.ComputationUsed) + }, + )) } func TestStorageUsed(t *testing.T) { diff --git a/fvm/transactionInvoker.go b/fvm/transactionInvoker.go index 57c0c449cbf..85d7375a0d3 100644 --- a/fvm/transactionInvoker.go +++ b/fvm/transactionInvoker.go @@ -397,21 +397,16 @@ func (executor *transactionExecutor) normalExecution() ( // Check if all account storage limits are ok // - // disable the computation/memory limit checks on storage checks, - // so we don't error from computation/memory limits on this part. - // // The storage limit check is performed for all accounts that were touched during the transaction. // The storage capacity of an account depends on its balance and should be higher than the accounts storage used. // The payer account is special cased in this check and its balance is considered max_fees lower than its // actual balance, for the purpose of calculating storage capacity, because the payer will have to pay for this tx. - executor.txnState.RunWithAllLimitsDisabled(func() { - err = executor.CheckStorageLimits( - executor.ctx, - executor.env, - bodySnapshot, - executor.proc.Transaction.Payer, - maxTxFees) - }) + err = executor.CheckStorageLimits( + executor.ctx, + executor.env, + bodySnapshot, + executor.proc.Transaction.Payer, + maxTxFees) if err != nil { return diff --git a/go.mod b/go.mod index 31e78ef695c..9efdb153d01 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.1 github.com/onflow/flow-core-contracts/lib/go/templates v0.15.1 github.com/onflow/flow-go-sdk v0.44.0 - github.com/onflow/flow/protobuf/go/flow v0.3.7 + github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 30163fef46a..085dd14509c 100644 --- a/go.sum +++ b/go.sum @@ -1370,8 +1370,8 @@ github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.7 h1:+6sBdlE/u4ZMTVB9U1lA6Xn2Bd48lOOX96Bv9dNubsk= -github.com/onflow/flow/protobuf/go/flow v0.3.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252 h1:W0xm80Qc5RkFJw7yQIj7OiMacCZw3et/tx/5N9rN2qk= +github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= github.com/onflow/sdks v0.5.0/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b h1:6O/BEmA99PDT5QVjoJgrYlGsWnpxGJTAMmsC+V9gyds= diff --git a/insecure/go.mod b/insecure/go.mod index 630703a567d..b7542dcf446 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -210,7 +210,7 @@ require ( github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 // indirect github.com/onflow/flow-go-sdk v0.44.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect - github.com/onflow/flow/protobuf/go/flow v0.3.7 // indirect + github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252 // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b // indirect github.com/onsi/ginkgo/v2 v2.13.2 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 905a4c307ed..d6c0d7ce470 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1333,8 +1333,8 @@ github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.7 h1:+6sBdlE/u4ZMTVB9U1lA6Xn2Bd48lOOX96Bv9dNubsk= -github.com/onflow/flow/protobuf/go/flow v0.3.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252 h1:W0xm80Qc5RkFJw7yQIj7OiMacCZw3et/tx/5N9rN2qk= +github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= github.com/onflow/sdks v0.5.0/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b h1:6O/BEmA99PDT5QVjoJgrYlGsWnpxGJTAMmsC+V9gyds= diff --git a/integration/go.mod b/integration/go.mod index 8c8dbfd74d7..439d4aaceed 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -25,11 +25,11 @@ require ( github.com/onflow/crypto v0.25.0 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.1 github.com/onflow/flow-core-contracts/lib/go/templates v0.15.1 - github.com/onflow/flow-emulator v0.58.1-0.20240130123529-733cc9417abc - github.com/onflow/flow-go v0.32.7 - github.com/onflow/flow-go-sdk v0.44.0 + github.com/onflow/flow-emulator v0.58.1-0.20240313174529-1d05170401b6 + github.com/onflow/flow-go v0.33.2-0.20240306234901-64ab8d27ea30 + github.com/onflow/flow-go-sdk v0.46.0 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 - github.com/onflow/flow/protobuf/go/flow v0.3.7 + github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252 github.com/plus3it/gorecurcopy v0.0.1 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/client_model v0.5.0 @@ -252,7 +252,6 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 // indirect - github.com/onflow/flow-go/crypto v0.25.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead // indirect github.com/onflow/sdks v0.5.0 // indirect diff --git a/integration/go.sum b/integration/go.sum index e539fdafd3e..1433f1af10b 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1414,21 +1414,19 @@ github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.1 h1:xF5wHug6H8vKfz github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.1/go.mod h1:WHp24VkUQfcfZi0XjI1uRVRt5alM5SHVkwOil1U2Tpc= github.com/onflow/flow-core-contracts/lib/go/templates v0.15.1 h1:EjWjbyVEA+bMxXbM44dE6MsYeqOu5a9q/EwSWa4ma2M= github.com/onflow/flow-core-contracts/lib/go/templates v0.15.1/go.mod h1:c09d6sNyF/j5/pAynK7sNPb1XKqJqk1rxZPEqEL+dUo= -github.com/onflow/flow-emulator v0.58.1-0.20240130123529-733cc9417abc h1:KwGHzWXsiYajQxc2FGgQDWVEityhrfQwXgEYmksohOA= -github.com/onflow/flow-emulator v0.58.1-0.20240130123529-733cc9417abc/go.mod h1:zYCPwOMqoJK+38+Pxzl1WmN5S1x4+Fh1UIOXVgiiOck= +github.com/onflow/flow-emulator v0.58.1-0.20240313174529-1d05170401b6 h1:07q2fysEezb3QGpRATEQZPG8Gz6exxlIe7l/yLagrn4= +github.com/onflow/flow-emulator v0.58.1-0.20240313174529-1d05170401b6/go.mod h1:kQcINHh4JJs3LG5OsMJ9gSfTkEMZnH41WwnWpz6NG4E= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 h1:B4ll7e3j+MqTJv2122Enq3RtDNzmIGRu9xjV7fo7un0= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= -github.com/onflow/flow-go-sdk v0.44.0 h1:gVRLcZ6LUNs/5mzHDx0mp4mEnBAWD62O51P4/nYm4rE= -github.com/onflow/flow-go-sdk v0.44.0/go.mod h1:mm1Fi2hiMrexNMwRzTrAN2zwTvlP8iQ5CF2JSAgJR8U= +github.com/onflow/flow-go-sdk v0.46.0 h1:mrIQziCDe6Oi5HH/aPFvYluh1XUwO6lYpoXLWrBZc2s= +github.com/onflow/flow-go-sdk v0.46.0/go.mod h1:azVWF0yHI8wT1erF0vuYGqQZybl6Frbc+0Zu3rIPeHc= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= -github.com/onflow/flow-go/crypto v0.25.0 h1:6lmoiAQ3APCF+nV7f4f2AXL3PuDKqQiWqRJXmjrMEq4= -github.com/onflow/flow-go/crypto v0.25.0/go.mod h1:OOb2vYcS8AOCajBClhHTJ0NKftFl1RQgTQ0+Vh4nbqk= github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.7 h1:+6sBdlE/u4ZMTVB9U1lA6Xn2Bd48lOOX96Bv9dNubsk= -github.com/onflow/flow/protobuf/go/flow v0.3.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252 h1:W0xm80Qc5RkFJw7yQIj7OiMacCZw3et/tx/5N9rN2qk= +github.com/onflow/flow/protobuf/go/flow v0.3.7-0.20240305102946-3efec6679252/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead h1:2j1Unqs76Z1b95Gu4C3Y28hzNUHBix7wL490e61SMSw= github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead/go.mod h1:E3ScfQb5XcWJCIAdtIeEnr5i5l2y60GT0BTXeIHseWg= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/integration/localnet/builder/bootstrap.go b/integration/localnet/builder/bootstrap.go index 1b6543f1a33..f41b5682b95 100644 --- a/integration/localnet/builder/bootstrap.go +++ b/integration/localnet/builder/bootstrap.go @@ -470,10 +470,12 @@ func prepareObserverService(i int, observerName string, agPublicKey string) Serv fmt.Sprintf("--secure-rpc-addr=%s:%s", observerName, testnet.GRPCSecurePort), fmt.Sprintf("--http-addr=%s:%s", observerName, testnet.GRPCWebPort), fmt.Sprintf("--rest-addr=%s:%s", observerName, testnet.RESTPort), + fmt.Sprintf("--state-stream-addr=%s:%s", observerName, testnet.ExecutionStatePort), "--execution-data-dir=/data/execution-data", "--execution-data-sync-enabled=true", "--execution-data-indexing-enabled=true", "--execution-state-dir=/data/execution-state", + "--event-query-mode=execution-nodes-only", ) service.AddExposedPorts( @@ -481,6 +483,7 @@ func prepareObserverService(i int, observerName string, agPublicKey string) Serv testnet.GRPCSecurePort, testnet.GRPCWebPort, testnet.RESTPort, + testnet.ExecutionStatePort, ) // observer services rely on the access gateway diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 52ec09614d2..5a4484c39be 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -783,6 +783,9 @@ func (net *FlowNetwork) addObserver(t *testing.T, conf ObserverConfig) { nodeContainer.exposePort(RESTPort, testingdock.RandomPort(t)) nodeContainer.AddFlag("rest-addr", nodeContainer.ContainerAddr(RESTPort)) + nodeContainer.exposePort(ExecutionStatePort, testingdock.RandomPort(t)) + nodeContainer.AddFlag("state-stream-addr", nodeContainer.ContainerAddr(ExecutionStatePort)) + nodeContainer.opts.HealthCheck = testingdock.HealthCheckCustom(nodeContainer.HealthcheckCallback()) suiteContainer := net.suite.Container(containerOpts) diff --git a/integration/tests/access/cohort3/execution_state_sync_test.go b/integration/tests/access/cohort3/execution_state_sync_test.go index 7d6118ef4b8..38a156549db 100644 --- a/integration/tests/access/cohort3/execution_state_sync_test.go +++ b/integration/tests/access/cohort3/execution_state_sync_test.go @@ -120,6 +120,7 @@ func (s *ExecutionStateSyncSuite) buildNetworkConfig() { AdditionalFlags: []string{ fmt.Sprintf("--execution-data-dir=%s", testnet.DefaultExecutionDataServiceDir), "--execution-data-sync-enabled=true", + "--event-query-mode=execution-nodes-only", }, }} diff --git a/integration/tests/access/cohort3/grpc_state_stream_test.go b/integration/tests/access/cohort3/grpc_state_stream_test.go index 3bb8614b70d..0614b78f2e2 100644 --- a/integration/tests/access/cohort3/grpc_state_stream_test.go +++ b/integration/tests/access/cohort3/grpc_state_stream_test.go @@ -72,6 +72,8 @@ func (s *GrpcStateStreamSuite) SetupTest() { testnet.WithAdditionalFlag("--execution-data-indexing-enabled=true"), testnet.WithAdditionalFlagf("--execution-state-dir=%s", testnet.DefaultExecutionStateDir), testnet.WithAdditionalFlag("--event-query-mode=local-only"), + testnet.WithAdditionalFlag("--supports-observer=true"), + testnet.WithAdditionalFlagf("--public-network-execution-data-sync-enabled=true"), ) controlANConfig := testnet.NewNodeConfig( flow.RoleAccess, @@ -104,7 +106,20 @@ func (s *GrpcStateStreamSuite) SetupTest() { controlANConfig, // access_2 } - conf := testnet.NewNetworkConfig("access_event_streaming_test", nodeConfigs) + // add the observer node config + observers := []testnet.ObserverConfig{{ + ContainerName: testnet.PrimaryON, + LogLevel: zerolog.DebugLevel, + AdditionalFlags: []string{ + fmt.Sprintf("--execution-data-dir=%s", testnet.DefaultExecutionDataServiceDir), + fmt.Sprintf("--execution-state-dir=%s", testnet.DefaultExecutionStateDir), + "--execution-data-sync-enabled=true", + "--event-query-mode=execution-nodes-only", + "--execution-data-indexing-enabled=true", + }, + }} + + conf := testnet.NewNetworkConfig("access_event_streaming_test", nodeConfigs, testnet.WithObservers(observers...)) s.net = testnet.PrepareFlowNetwork(s.T(), conf, flow.Localnet) // start the network @@ -124,12 +139,19 @@ func (s *GrpcStateStreamSuite) TestHappyPath() { sdkClientControlAN, err := getClient(controlANURL) s.Require().NoError(err) + testONURL := fmt.Sprintf("localhost:%s", s.net.ContainerByName(testnet.PrimaryON).Port(testnet.ExecutionStatePort)) + sdkClientTestON, err := getClient(testONURL) + s.Require().NoError(err) + time.Sleep(20 * time.Second) - testEvents, testErrs, err := SubscribeEventsByBlockHeight(s.ctx, sdkClientTestAN, 0, &executiondata.EventFilter{}) + testANEvents, testANErrs, err := SubscribeEventsByBlockHeight(s.ctx, sdkClientTestAN, 0, &executiondata.EventFilter{}) + s.Require().NoError(err) + + controlANEvents, controlANErrs, err := SubscribeEventsByBlockHeight(s.ctx, sdkClientControlAN, 0, &executiondata.EventFilter{}) s.Require().NoError(err) - controlEvents, controlErrs, err := SubscribeEventsByBlockHeight(s.ctx, sdkClientControlAN, 0, &executiondata.EventFilter{}) + testONEvents, testONErrs, err := SubscribeEventsByBlockHeight(s.ctx, sdkClientTestON, 0, &executiondata.EventFilter{}) s.Require().NoError(err) txCount := 10 @@ -161,28 +183,37 @@ func (s *GrpcStateStreamSuite) TestHappyPath() { targetEvent := flow.EventType("flow.AccountCreated") - foundTxCount := 0 + foundANTxCount := 0 + foundONTxCount := 0 r := newResponseTracker() for { select { - case err := <-testErrs: + case err := <-testANErrs: s.Require().NoErrorf(err, "unexpected test AN error") - case err := <-controlErrs: + case err := <-controlANErrs: s.Require().NoErrorf(err, "unexpected control AN error") - case event := <-testEvents: + case err := <-testONErrs: + s.Require().NoErrorf(err, "unexpected test ON error") + case event := <-testANEvents: if has(event.Events, targetEvent) { - s.T().Logf("adding test events: %d %d %v", event.BlockHeight, len(event.Events), event.Events) - r.Add(s.T(), event.BlockHeight, "test", &event) - foundTxCount++ + s.T().Logf("adding access test events: %d %d %v", event.BlockHeight, len(event.Events), event.Events) + r.Add(s.T(), event.BlockHeight, "access_test", &event) + foundANTxCount++ } - case event := <-controlEvents: + case event := <-controlANEvents: if has(event.Events, targetEvent) { s.T().Logf("adding control events: %d %d %v", event.BlockHeight, len(event.Events), event.Events) - r.Add(s.T(), event.BlockHeight, "control", &event) + r.Add(s.T(), event.BlockHeight, "access_control", &event) + } + case event := <-testONEvents: + if has(event.Events, targetEvent) { + s.T().Logf("adding observer test events: %d %d %v", event.BlockHeight, len(event.Events), event.Events) + r.Add(s.T(), event.BlockHeight, "observer_test", &event) + foundONTxCount++ } } - if foundTxCount >= txCount { + if foundANTxCount >= txCount && foundONTxCount >= txCount { break } } @@ -208,21 +239,24 @@ func (r *ResponseTracker) Add(t *testing.T, blockHeight uint64, name string, eve } r.r[blockHeight][name] = *events - if len(r.r[blockHeight]) != 2 { + if len(r.r[blockHeight]) != 3 { return } - err := r.compare(t, r.r[blockHeight]) + err := r.compare(t, r.r[blockHeight]["access_control"], r.r[blockHeight]["access_test"]) if err != nil { - log.Fatalf("failure comparing %d: %v", blockHeight, err) + log.Fatalf("failure comparing access and access data %d: %v", blockHeight, err) } + + err = r.compare(t, r.r[blockHeight]["access_control"], r.r[blockHeight]["observer_test"]) + if err != nil { + log.Fatalf("failure comparing access and observer data %d: %v", blockHeight, err) + } + delete(r.r, blockHeight) } -func (r *ResponseTracker) compare(t *testing.T, data map[string]flow.BlockEvents) error { - controlData := data["control"] - testData := data["test"] - +func (r *ResponseTracker) compare(t *testing.T, controlData flow.BlockEvents, testData flow.BlockEvents) error { require.Equal(t, controlData.BlockID, testData.BlockID) require.Equal(t, controlData.BlockHeight, testData.BlockHeight) require.Equal(t, len(controlData.Events), len(testData.Events)) @@ -261,7 +295,6 @@ func SubscribeEventsByBlockHeight( Filter: filter, HeartbeatInterval: 1, } - stream, err := client.SubscribeEvents(ctx, req) if err != nil { return nil, nil, err diff --git a/model/flow/block.go b/model/flow/block.go index abd62ff8595..28daab74619 100644 --- a/model/flow/block.go +++ b/model/flow/block.go @@ -2,7 +2,10 @@ package flow -import "fmt" +import ( + "fmt" + "time" +) func Genesis(chainID ChainID) *Block { @@ -111,3 +114,28 @@ func (b *CertifiedBlock) View() uint64 { func (b *CertifiedBlock) Height() uint64 { return b.Block.Header.Height } + +// BlockDigest holds lightweight block information which includes only block id, block height and block timestamp +type BlockDigest struct { + id Identifier + Height uint64 + Timestamp time.Time +} + +// NewBlockDigest constructs a new block digest. +func NewBlockDigest( + id Identifier, + height uint64, + timestamp time.Time, +) *BlockDigest { + return &BlockDigest{ + id: id, + Height: height, + Timestamp: timestamp, + } +} + +// ID returns the id of the BlockDigest. +func (b *BlockDigest) ID() Identifier { + return b.id +} diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index f77bf321854..e753f8dddad 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -18,7 +18,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" - "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/blobs" @@ -408,7 +408,7 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun suite.downloader = mockDownloader(cfg.executionDataEntries) suite.distributor = requester.NewExecutionDataDistributor() - heroCache := herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, logger, metrics) + heroCache := herocache.NewBlockExecutionData(subscription.DefaultCacheSize, logger, metrics) cache := cache.NewExecutionDataCache(suite.downloader, headers, seals, results, heroCache) followerDistributor := pubsub.NewFollowerDistributor() diff --git a/module/state_synchronization/requester/jobs/execution_data_reader_test.go b/module/state_synchronization/requester/jobs/execution_data_reader_test.go index da6b515c72b..e4545aebee3 100644 --- a/module/state_synchronization/requester/jobs/execution_data_reader_test.go +++ b/module/state_synchronization/requester/jobs/execution_data_reader_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" @@ -92,8 +91,9 @@ func (suite *ExecutionDataReaderSuite) reset() { ) suite.downloader = new(exedatamock.Downloader) + var executionDataCacheSize uint32 = 100 // Use local value to avoid cycle dependency on subscription package - heroCache := herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, unittest.Logger(), metrics.NewNoopCollector()) + heroCache := herocache.NewBlockExecutionData(executionDataCacheSize, unittest.Logger(), metrics.NewNoopCollector()) cache := cache.NewExecutionDataCache(suite.downloader, suite.headers, suite.seals, suite.results, heroCache) suite.reader = NewExecutionDataReader(